{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import torch as t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ReLU + Sigmoid + Cross Entropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net1(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net1, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(28*28, 30)\n",
    "        self.fc2 = nn.Linear(30, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.logsigmoid(self.fc2(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mnist_loader import load_data_shared, vectorized_result\n",
    "training_data, validation_data, test_data = load_data_shared(filename=\"../mnist.pkl.gz\",\n",
    "                                                                     seed=666,\n",
    "                                                                     train_size=2000,\n",
    "                                                                     vali_size=0,\n",
    "                                                                     test_size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(data, net):\n",
    "    with t.no_grad():\n",
    "        #for index in range(test_data[0].shape[0]):\n",
    "            # get the inputs\n",
    "        inputs, labels = t.Tensor(data[0]), data[1]\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        _, predicted = t.max(outputs, 1)\n",
    "\n",
    "        #print('Predicted: ', predicted)\n",
    "        #print('target: ', t.Tensor(labels).int())\n",
    "\n",
    "        correct = (predicted == t.Tensor(labels)).sum().item()\n",
    "        accuracy = correct / data[0].shape[0]\n",
    "        return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit(net, criterion, optimizer):\n",
    "    loss_scores = []\n",
    "    test_scores = []\n",
    "    train_scores = []\n",
    "    for epoch in range(100):  # loop over the dataset multiple times\n",
    "\n",
    "        # get the inputs\n",
    "        inputs, labels = t.Tensor(training_data[0]), t.Tensor(training_data[1])\n",
    "        vector_labels = t.Tensor([vectorized_result(y) for y in training_data[1]])\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels.long())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        loss_scores.append(loss.item())\n",
    "        train_scores.append(predict(training_data, net))\n",
    "        test_scores.append(predict(test_data, net))\n",
    "    print('Finished Training')\n",
    "    return loss_scores, train_scores, test_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net1 = Net1()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net1.parameters(), lr = 1e-1)\n",
    "loss_scores1, train_scores1, test_scores1 = fit(net1, criterion, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxU5b3H8c9vMlkIJCxZ2AMEAkFAtiA7KlQB9YJaq6JVa1XqUrXWWq29t/XW29Vq3UDEpWhV6obF3VIV2dGw70sCSNiyQVgCZHvuH4leLiUkkAknM/N9v155wcw5zvkeD349PPPMM+acQ0REgp/P6wAiIhIYKnQRkRChQhcRCREqdBGREKFCFxEJEX6vDpyYmOg6duzo1eFFRILSkiVL8p1zSSfa5lmhd+zYkczMTK8OLyISlMxsW3XbNOQiIhIiVOgiIiFChS4iEiJU6CIiIUKFLiISIlToIiIhQoUuIhIigq7Qt+Yf4tF/bmDepnyKS8q8jiMi0mB49sGi07VyRxGTZ2fx1Geb8fuM3u2bMSi1BYNSE+jfoTmxUUF3SiIiAWFefcFFRkaGO91Pih48Wkbm1kIWbylkcXYBK3OKKKtw3xb84NQEBneuLPiYyIgAJxcR8Y6ZLXHOZZxwWzAW+vEOHS1jyba9LMwuYGFWAat2FFFe4YiK8NE3pRmDOycwODWBPinNiPar4EUkeIV8oR/vwJFSMrf+X8Gv3lmEcxDt99G/Q3MGVd3B927XjCh/0L2NICJhLOwK/XhFxaUs3lLAouxCFmYXsG7XfgBiIisLfmCnBAalJtC7fVPdwYtIgxb2hX68vYdK+HJrIQuzCli8pZD1u/f/2x28Cl5EGiIVeg32FZdUvcFayKLsAtadoOA1RCMiDYEK/RTtKy7hq617WVQ1Bv9NwcdE+sjo0ILBnRMY0jmBXm2b4o9QwYvImaNCr6Nv7uAXZhWwKLuA9bsPANAk2s/ATt8UfCLpreLw+czjtCISyk5W6PoUTi00i41idI9WjO7RCoCCg0dZlF3Igqx8FmYV8On6XABaNI76dg780C6JdEyIxUwFLyJnhu7QA2BX0WEWbC5gQVYBC7Ly2VV0BIA2TWMY0iWRYV0SGdIlgeS4GI+Tikiwq9OQi5m1B14GWgEVwFTn3BPH7XMtcH/Vw4PAbc65FSd73VAq9GM559iSf+jbcl+QVcC+4lIAurZswtAuiQxPS2RgpwQaR+svSCJyaupa6K2B1s65pWYWBywBLnXOrT1mnyHAOufcXjMbCzzknBt4stcN1UI/XkWFY+2u/czbnM+8Tfl8ubWQkrIK/D6jX0pzhqUlMiwtkbP1BquI1EJA3xQ1s5nA0865WdVsbw6sds61PdnrhEuhH+9IaTlLtu1l7qZ85m3OY83Oyhk0cTF+hnROYHhaEiPSkkhJiPU6qog0QAErdDPrCMwBejrn9lezz8+AdOfczSfYNhGYCJCSktJ/27ZttT52qCo8VMKCrMq797mb8tmx7zAAHRJiGZ6WyLAuSQzpkkB8TKTHSUWkIQhIoZtZE+AL4LfOuRnV7HM+MBkY5pwrONnrhesd+sk458jOP1RV7nkszCrgUEk5ET6jT/tmDE9LZETXJHq3a0aEpkeKhKU6F7qZRQLvA5845x6rZp+zgXeAsc65jTW9pgq9ZqXlFSz7eh9zN+UxZ2MeK3dULjIWH+NnWFoiI9KSGNE1iTbNGnkdVUTOkLq+KWrAS0Chc+4n1eyTAnwGXO+cW1CbUCr0U7f3UAnzs/KZszGPORvz2b2/cnpkWnITRnStLPeBnVpoDXiREFbXQh8GzAVWUTltEeBBIAXAOTfFzJ4Hvgt8MyheVt0Bv6FCrxvnHJtyDzJnYx5fbMxj8ZbK2TPRfh+DUhM4t2sS53ZLIjWxsT7cJBJC9NH/MHC4pJxFWwq+LfjsvEMAtG/RqLLcuyYzpLPmvosEOxV6GNpeWMwXG/OYvSGPBVn5FJeUExXh45xOLTivWxLndUumc5Lu3kWCjQo9zJWUVZC5tZDZG/P4fH0um3IPApV37+d3S+b89GQGpyZo7F0kCKjQ5f/J2VvM7A15zN6Qy/zNBRwuLScm0seQzomcn57MyPRk2mrmjEiDpEKXah0pLWfxlkI+X5/LZ+tz+bqwGID0VnGM6p7MyPSW9Gmvee8iDYUKXWrFOUdW3iE+W7+HT9flkrltL+UVjoTGUYxMT+Y7Z7VkeFoisVF6Y1XEKyp0OS1FxaXM3pjLv9blMntDLgeOlBHl9zGsSyIXnNWSUd2TtSSwyBmmL7iQ09I0NpLxfdoyvk9bSssr+GpLIbPW7WHW2j18tj4XM+iX0pwLz2rJ6B6t6JjY2OvIImFNd+hyypxzrN99gH+u2cM/1+5mzc7Kddq6tYxjdM9WjO3ZivRWcZoSKVIPNOQi9SpnbzGfrNnDJ2t289XWQpyrXC1ybM/WXNSrFb3aNlW5iwSICl3OmLwDR5m1dg8fr9nNgs35lFU42jVvxEW9WnNxr9ac3U7lLlIXKnTxxL7iEv65dg8frdrF3E3/V+6XnN2GS85uTY828Sp3kVOkQhfPFRWX8sna3XywchfzNudTXuFITWzMf/Ruw7g+beic1MTriCJBQYUuDUrhoRI+Xr2b91bsZNGWApyDnm3jubRPW8b1bkNyvKZCilRHhS4N1u6iI7y/cifvrtjJypwifAZDuyRyWd+2jO7RSqtDihxHhS5BISvvIDOX7WDGsh3k7D1MbFQEY3q24or+7RjUKQGflh8QUaFLcKmocGRu28s7y3J4f8UuDhwto13zRny3Xzuu6N+O9i1ivY4o4hkVugStI6XlfLJmN28tyWHe5nwAhnVJ5MqM9lzYoyXRfi35K+FFhS4hIWdvMW8tyeHNzBx27DtMi8ZRXNG/HVcPaE+qZslImFChS0ipqHDM25zP9C+/ZtbaPZRVOAanJnDd4A5ccFZLIiN8XkcUqTcqdAlZuQeO8GZmDq8t/pod+w6THBfNhHNSuGZgCi01/VFCkApdQl55heOLjbn8beE2Zm/MI8KMsb1a84MhHemX0kyfSJWQUaflc82sPfAy0AqoAKY65544bh8DngAuAoqBHzjnltY1uEhtRfiMkektGZnekq35h/jbom28kbmd91bspHe7pvxwWCcu6tVawzES0mq8Qzez1kBr59xSM4sDlgCXOufWHrPPRcCdVBb6QOAJ59zAk72u7tClvh06WsaMpTn8df5WsvMP0bppDDcO7cjV56QQHxPpdTyR0xLQIRczmwk87ZybdcxzzwKznXPTqx5vAM5zzu2q7nVU6HKmVFQ4Zm/M5bk5W1iYXUCTaD/XDEzhpmGdNM4uQSdg31hkZh2BvsDi4za1BbYf8zin6rn/V+hmNhGYCJCSknIqhxY5bb5jhmNW5RQxdW42z8/NZtr8rVzery0/OrcznfRtSxICaj2gaGZNgLeBnzjn9h+/+QT/yL/d+jvnpjrnMpxzGUlJSaeWVCQAerVrylMT+jL7Z+dz5YB2zFi2g1GPzuau6cvYuOeA1/FE6qRWhW5mkVSW+avOuRkn2CUHaH/M43bAzrrHE6kfKQmx/M+lvZh3//ncMjyVf63bw4V/mcNtryxh/e7j71dEgkONhV41g+UFYJ1z7rFqdnsXuN4qDQKKTjZ+LtJQJMfF8IuLujP//pHcObIL8zblM+bxudzx6lI26Y5dgkxtZrkMA+YCq6ictgjwIJAC4JybUlX6TwNjqJy2eKNz7qTveOpNUWmI9hWX8PzcLfx1/haKS8u5rE9b7rmgqxYEkwZDHywSOUV7D5Uw5Ysspi3YSoVzXD0ghbtGpZEUF+11NAlzKnSR07Rn/xGe/HQTr3+1nWi/j4kjOnPz8E764g3xjApdpI6y8w7yyCcb+Gj1bpLiorlvdDeu6NdOX7ohZ9zJCl2fgxaphdSkJjzz/f68fdsQ2jVvxM/fWsm4SfNYnF3gdTSRb6nQRU5B/w7NmXHbEJ64ug+FB0u4auoi7pq+jN1FR7yOJqJCFzlVZsb4Pm359N7zuHtUGh+v2c3IR2cz5YssSssran4BkXqiQhc5TY2iIrjngq78655zGdI5kT98tJ6Ln5xL5tZCr6NJmFKhi9RRSkIsz9+QwQs3ZHDoaDlXTFnIA2+vpKi41OtoEmZU6CIBMqp7S2b9dAQTR6Ty5pIcvvOXL/h49W6vY0kYUaGLBFBslJ8HL+rOzDuGktQkmltfWcLtry4h78BRr6NJGFChi9SDnm2bMvPHQ7lvdDf+tS6X0Y/P4cNVWt5I6pcKXaSeREb4uOP8Lnxw5zDaNmvE7a8u5e6/L9PYutQbFbpIPUtrGceM24fw0wu68sHKXYx5Yg4Ls/SBJAk8FbrIGRAZ4eOuUWm8c/tQGkVGcM3zi/jDR+spKdO8dQkcFbrIGdSrXVPev2sYVw9oz5QvsvjeswvZXljsdSwJESp0kTMsNsrP7y8/m8nX9iM79yAXPzmXT9ZoeqPUnQpdxCMX9WrNB3cNp2NiY370tyU8/P5aLR0gdaJCF/FQSkIsb946mB8M6cgL87Zw7XOLyT2ghb7k9KjQRTwW7Y/goXE9ePyqPqzcsY9LnpzHkm1aD0ZOnQpdpIG4tG/bylkwURFMmLqYNzK3ex1JgowKXaQB6d46npl3DOWcTi34+Vsrefj9tZRpXF1qqcZCN7MXzSzXzFZXs72pmb1nZivMbI2Z3Rj4mCLho1lsFNNuHPDtuPpNL2Vy4Ig+XSo1q80d+jRgzEm23wGsdc71Bs4DHjWzqLpHEwlf/ggfD43rwe8v78W8zfl8b8pCdu477HUsaeBqLHTn3BzgZO/QOCDOzAxoUrVvWWDiiYS3CeekMO3GAezYe5hLJ81n9Y4iryNJAxaIMfSnge7ATmAVcLdz7oSDfmY20cwyzSwzLy8vAIcWCX3D05J487bB+H3GVc8uZN6mfK8jSQMViEIfDSwH2gB9gKfNLP5EOzrnpjrnMpxzGUlJSQE4tEh4SG8Vzzt3DKV9i1hunPYl767Y6XUkaYACUeg3AjNcpc3AFiA9AK8rIsdoGR/D6z8aTN+U5tw1fRnT5m/xOpI0MIEo9K+BUQBm1hLoBmQH4HVF5DhNG0Xy8g/PYXSPljz03lqe+nQTzjmvY0kD4a9pBzObTuXslUQzywF+DUQCOOemAA8D08xsFWDA/c45DfKJ1JOYyAgmXdOPn7+9kkdnbeRgSRkPjEmncl6ChLMaC905N6GG7TuBCwOWSERq5I/w8ecrehMbFcGzX2RTfLSc/x7XA59PpR7Oaix0EWmYfD7j4fE9aRzl59k52VQ4x8Pje6rUw5gKXSSImRkPjE0nwmdMnp2FA/5HpR62VOgiQc7MuG90N8xg0udZOAe/vVSlHo5U6CIhwMz42YXdgMpSj/b7+PV/nKU3SsOMCl0kRHxT6kdLK3h+3hYaRUXw89HdVOphRIUuEkLMjF9e3J3DpeU8MzuL2MgI7hyV5nUsOUNU6CIhxqxy9svh0nIenbWR+EaR3DCko9ex5AxQoYuEIJ/P+NN3z+bAkTIeem8NzRtHMa53G69jST3TNxaJhCh/hI+nJvRlQMcW3PvGcuZs1AqnoU6FLhLCYiIjeP6GDLokx3HrK0tYmbPP60hSj1ToIiEuPiaSl344gBaNo/jhtK/YXljsdSSpJyp0kTCQHBfDtBvPobTcccNfv2RfcYnXkaQeqNBFwkSX5CY8d30GOYWHueXlTI6UlnsdSQJMhS4SRs7p1ILHrurNV1v3cv/bK7WWeojRtEWRMHPJ2W3YVlDMI59soHNSE+7SB49ChgpdJAzdfl5nsvIO8tisjXRKbMx/aI56SNCQi0gYMjN+f3kvBnRszr1vrmD5dk1nDAUqdJEwFe2P4NnrMmgZH83ElzPZs/+I15GkjlToImGsReMonrs+g4NHy5j4tyWa+RLkVOgiYS69VTyPXdmbFdv38eA7qzTzJYip0EWEMT1bc/eoNGYs3cGL87d6HUdOU42FbmYvmlmuma0+yT7nmdlyM1tjZl8ENqKInAl3j0rjgrNa8rsP17Eou8DrOHIaanOHPg0YU91GM2sGTAbGOed6AN8LTDQROZN8PuPRK3vToUUsP35tKbuKDnsdSU5RjYXunJsDFJ5kl2uAGc65r6v2zw1QNhE5w+JjIpl6fX8Ol5Rz2ytLOVqmN0mDSSDG0LsCzc1stpktMbPrq9vRzCaaWaaZZeblaW1mkYaoS3Icj17Zm+Xb9/Gb99Z6HUdOQSAK3Q/0By4GRgP/ZWZdT7Sjc26qcy7DOZeRlJQUgEOLSH0Y07M1Pzo3lVcXf82MpTlex5FaCkSh5wAfO+cOOefygTlA7wC8roh46L4LuzGwUwsefGcV63fv9zqO1EIgCn0mMNzM/GYWCwwE1gXgdUXEQ/4IH09d05e4mEhue2UpB46Ueh1JalCbaYvTgYVANzPLMbObzOxWM7sVwDm3DvgYWAl8CTzvnKt2iqOIBI/kuBientCXrwuLtdxuEKhxtUXn3IRa7PMI8EhAEolIgzIwNYH7RnfjDx+t56UFW/nB0E5eR5Jq6JOiIlKjicNTGZmezG8/XMcKrczYYKnQRaRGPp/x6Pd6kxwXwx2vLaWoWOPpDZEKXURqpXnjKJ66pi+7i47ws7dWaDy9AVKhi0it9UtpzgNj05m1dg/TFmz1Oo4cR4UuIqfkpmGd+E73ZH734TpW5mg8vSFRoYvIKTEz/vy93iQ1iebHry1jv+anNxgqdBE5Zc1iK8fTd+w7zC9m6EsxGgoVuoiclv4dWnDvhV35YOUupn+53es4ggpdROrg1hGdGZ6WyH+/t0brvTQAKnQROW0+n/HYlX2IbxTJj19bRnFJmdeRwpoKXUTqJCkumsev6kNW3kEeeneN13HCmgpdROpsaJdE7jivC29k5jBz+Q6v44QtFbqIBMRPvpNGRofm/PKd1WzNP+R1nLCkQheRgPBH+HhiQl8ifMad05dRUlbhdaSwo0IXkYBp26wRf7ribFbtKOKPH6/3Ok7YUaGLSECN7tGKHwzpyAvztvDpuj1exwkrKnQRCbgHxqZzVut47n1zBbuKDnsdJ2yo0EUk4GIiI3j6mr6UlFVw9/TllJVrPP1MUKGLSL1ITWrCby/ryZdbC3ny001exwkLKnQRqTeX9W3Hd/u146nPN7Ngc77XcUJejYVuZi+aWa6Zra5hvwFmVm5mVwQunogEu9+M70FqYmPufn05eQeOeh0npNXmDn0aMOZkO5hZBPBH4JMAZBKRENI42s+ka/ux/3Ap97y+nIoKLbVbX2osdOfcHKCwht3uBN4GcgMRSkRCS3qreB4a14N5m/OZPHuz13FCVp3H0M2sLXAZMKUW+040s0wzy8zLy6vroUUkiFw9oD3jerfhsVkbWZxd4HWckBSIN0UfB+53zpXXtKNzbqpzLsM5l5GUlBSAQ4tIsDAzfnd5LzokNObO6cvIP6jx9EALRKFnAH83s63AFcBkM7s0AK8rIiGmSbSfSdf0o0jj6fWizoXunOvknOvonOsIvAXc7pz7R52TiUhIOqtN5Xj63E35TPpc4+mB5K9pBzObDpwHJJpZDvBrIBLAOVfjuLmIyPGuHtCexdkF/OVfG+nfoTlDuiR6HSkkmFff1p2RkeEyMzM9ObaIeO/Q0TLGT5rPvuISPrhrOC3jY7yOFBTMbIlzLuNE2/RJURHxRONoP1O+34/iknJ+/NpSSrXeS52p0EXEM12S4/j95b34aute/vzJBq/jBD0Vuoh4anyftlw3qAPPzsnm49W7vI4T1FToIuK5/7ykO73bN+Nnb65kc+5Br+MELRW6iHgu2h/BM9f2I8rv49ZXlnDoaJnXkYKSCl1EGoQ2zRrx1IS+ZOcd5OdvrcSrGXjBTIUuIg3G0C6J3Dc6nQ9W7eK5udlexwk6KnQRaVBuPTeVi3q14g8frWfuJi3idypU6CLSoJgZj1zRmy7JTbhz+jK2FxZ7HSloqNBFpMFpHO1n6nUZVFQ4bnk5k+ISvUlaGyp0EWmQOiY25skJfdmw5wA/e3OF3iStBRW6iDRY53VL5oEx6Xy4ajdPfaaVGWtS42qLIiJemjgilQ27D/DYrI10bRnHmJ6tvI7UYOkOXUQatG++6ahP+2b89I3lrN253+tIDZYKXUQavJjICKZe15+mjSK5+aWvyN1/xOtIDZIKXUSCQnJ8DM/fkMG+w6Xc8nImh0tq/BrjsKNCF5Gg0aNNUx6/qg8rdxRx75v6TtLjqdBFJKhc2KMVvxhbOfPlz//UGurH0iwXEQk6twxPZUt+MZNnZ9GueSzXDEzxOlKDoEIXkaBjZjw8vge7ig7zXzNX07pZDOd3S/Y6luc05CIiQckf4ePpa/qR3iqOH7+6lNU7iryO5LkaC93MXjSzXDNbXc32a81sZdXPAjPrHfiYIiL/rkm0nxd/MICmjSK5cdpXYb+QV23u0KcBY06yfQtwrnPubOBhYGoAcomI1ErL+Bhe+uE5lJRVcP2LX1Jw8KjXkTxTY6E75+YAhSfZvsA5t7fq4SKgXYCyiYjUSlrLOF64IYOd+w7zw5fCd3XGQI+h3wR8VN1GM5toZplmlpmXp4XrRSRwMjq24KkJfVmVs49bX1lKSVmF15HOuIAVupmdT2Wh31/dPs65qc65DOdcRlJSUqAOLSICVM5R//3lvZizMY+fvrGc8jD74FFApi2a2dnA88BY51xBIF5TROR0XDUghaLDpfzuw/XEN4rkt5f2xMy8jnVG1LnQzSwFmAFc55zbWPdIIiJ1M3FEZ/YVlzJ5dhbxMZHcP6ZbWJR6jYVuZtOB84BEM8sBfg1EAjjnpgC/AhKAyVX/wsqccxn1FVhEpDbuG92N/UdKmfJFFrFREdw1Ks3rSPWuxkJ3zk2oYfvNwM0BSyQiEgBmxm/G9eRwSQWPzdpIo8gIbhmR6nWseqWP/otIyPL5jD9+txdHysr57YfriI70cf3gjl7HqjcqdBEJaf4IH49f1YejpRX8auYafGZ8f1AHr2PVC63lIiIhLzLCx6Rr+zIqPZn//MdqXlv8tdeR6oUKXUTCQrQ/gsnf78fI9GQefGdVSJa6Cl1Ewka0P4Jnvt+P87sl8eA7q3h54VavIwWUCl1Ewkq0P4Ip1/XnO91b8quZa3h+brbXkQJGhS4iYSfaH8Hka/sxtmcr/ueDdUyevdnrSAGhQheRsBTl9/HUhL6M79OGP328gT99vB7ngnvtF01bFJGw5Y/w8diVfYiN8jN5dhb7j5Tym3E98fmCc5kAFbqIhLUIn/G7y3oSH+Pn2TnZHDxSxiPf601kRPANYKjQRSTsmRkPjE0nvlEkj3yygX2HS5l8bT9io4KrIoPvf0EiIvXAzLjj/C7frqd+zXOL2XuoxOtYp0SFLiJyjAnnpPDM9/uzdtd+rpiyIKi+eFqFLiJynNE9WvHKTQPJO3CUyyYvYFVOkdeRakWFLiJyAud0asGM24cQ7fdx5bML+Wz9Hq8j1UiFLiJSjS7JcbxzxxA6Jzfm5pcyeWnBVq8jnZQKXUTkJJLjYnh94mBGprfk1++u4VczV1NWXuF1rBNSoYuI1KBxtJ9nr+vPxBGpvLxwGze9lMn+I6Vex/o3KnQRkVqI8BkPXtSd31/ei/mb87l00nyy8w56Hev/UaGLiJyCCeek8OrNA9lXXMr4SfP5YmOe15G+VWOhm9mLZpZrZqur2W5m9qSZbTazlWbWL/AxRUQajoGpCcy8YyhtmzXixr9+yZQvshrEwl61uUOfBow5yfaxQFrVz0TgmbrHEhFp2Nq3iGXG7UMY26s1f/hoPbe/upSDR8s8zVRjoTvn5gCFJ9llPPCyq7QIaGZmrQMVUESkoYqN8vP0hL788qLufLJmN5dOms/mXO/G1QMxht4W2H7M45yq5/6NmU00s0wzy8zLazjjTiIip8vMuGVEKq/cNJC9h0oY9/Q8Zi7f4UmWQBT6iRYOPuFgknNuqnMuwzmXkZSUFIBDi4g0DEO6JPLBXcM5q3U8d/99Of/5j1UcLSs/oxkCUeg5QPtjHrcDdgbgdUVEgkqrpjFMnziIW4Z34pVFX3P55AVsyT90xo4fiEJ/F7i+arbLIKDIObcrAK8rIhJ0IiN8/PLis3ju+gx27DvMJU/O5Z1lOWfk2LWZtjgdWAh0M7McM7vJzG41s1urdvkQyAY2A88Bt9dbWhGRIHHBWS358K7h9GjTlHteX8FPX1/OgXr+dKl5NXcyIyPDZWZmenJsEZEzpay8gqc+28xTn22ibfNGPH5VX/p3aH7ar2dmS5xzGSfapk+KiojUI3+Ej3su6MobPxqMc3Dlswt5Yd6WejmWCl1E5AzI6NiCD+8ezvjebUhNbFwvxwiub0AVEQli8TGRPHZVn3p7fd2hi4iECBW6iEiIUKGLiIQIFbqISIhQoYuIhAgVuohIiFChi4iECBW6iEiI8GwtFzPLA7ad5j+eCOQHME6wCMfzDsdzhvA873A8Zzj18+7gnDvhF0p4Vuh1YWaZ1S1OE8rC8bzD8ZwhPM87HM8ZAnveGnIREQkRKnQRkRARrIU+1esAHgnH8w7Hc4bwPO9wPGcI4HkH5Ri6iIj8u2C9QxcRkeOo0EVEQkTQFbqZjTGzDWa22cwe8DpPfTCz9mb2uZmtM7M1ZnZ31fMtzGyWmW2q+vX0v5iwATOzCDNbZmbvVz3uZGaLq877dTOL8jpjIJlZMzN7y8zWV13zweFwrc3snqo/36vNbLqZxYTitTazF80s18xWH/PcCa+vVXqyqt9Wmlm/UzlWUBW6mUUAk4CxwFnABDM7y9tU9aIMuNc51x0YBNxRdZ4PAJ8659KAT6seh6K7gXXHPP4j8Jeq894L3ORJqvrzBPCxcy4d6E3luYf0tTaztsBdQIZzricQAVxNaF7racCY456r7vqOBdKqfiYCz5zKgYKq0IFzgM3OuWznXAnwd2C8x5kCzjm3yzm3tOr3B6j8D7wtlef6UtVuLwGXepOw/phZO+Bi4PmqxwaMBN6q2iWkztvM4oERwAsAzrkS59w+wuBaU/kVmI3MzA/EArsIwWvtnJsDFB73dHXXdzzwsiSICIQAAAI0SURBVKu0CGhmZq1re6xgK/S2wPZjHudUPReyzKwj0BdYDLR0zu2CytIHkr1LVm8eB34OVFQ9TgD2OefKqh6H2jVPBfKAv1YNMz1vZo0J8WvtnNsB/Bn4msoiLwKWENrX+ljVXd86dVywFbqd4LmQnXdpZk2At4GfOOf2e52nvpnZJUCuc27JsU+fYNdQuuZ+oB/wjHOuL3CIEBteOZGqMePxQCegDdCYyuGG44XSta6NOv15D7ZCzwHaH/O4HbDToyz1yswiqSzzV51zM6qe3vPNX7+qfs31Kl89GQqMM7OtVA6njaTyjr1Z1V/LIfSueQ6Q45xbXPX4LSoLPtSv9XeALc65POdcKTADGEJoX+tjVXd969RxwVboXwFpVe+ER1H5Jsq7HmcKuKpx4xeAdc65x47Z9C5wQ9XvbwBmnuls9ck59wvnXDvnXEcqr+1nzrlrgc+BK6p2C6nzds7tBrabWbeqp0YBawnxa03lUMsgM4ut+vP+zXmH7LU+TnXX913g+qrZLoOAom+GZmrFORdUP8BFwEYgC/il13nq6RyHUfnXrJXA8qqfi6gcT/4U2FT1awuvs9bjv4PzgPerfp8KfAlsBt4Eor3OF+Bz7QNkVl3vfwDNw+FaA/8NrAdWA38DokPxWgPTqXyfoJTKO/Cbqru+VA65TKrqt1VUzgKq9bH00X8RkRARbEMuIiJSDRW6iEiIUKGLiIQIFbqISIhQoYuIhAgVuohIiFChi4iEiP8FBudzaDl9hJMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXwV9bnH8c+TPSSBAAl7AoGEVVYjuG+A4l7rBmjV1pZre2nV9rbX1ta29rZVa63a0gWXqhXFXVGRRcVdlLCTsCSEJQsJSQhkI+t57h9zlBATcgInOTnnPO/XixeZmV/mPJOJX4ffzPx+oqoYY4zxfyG+LsAYY4x3WKAbY0yAsEA3xpgAYYFujDEBwgLdGGMCRJivPjghIUGHDRvmq483xhi/tHbt2lJVTWxtm88CfdiwYWRkZPjq440xxi+JyJ62tlmXizHGBAgLdGOMCRAW6MYYEyAs0I0xJkBYoBtjTICwQDfGmABhgW6MMQHCZ8+hG2OM3zi4FzY8C64m7+xv1CwYfLJ39tWMBboxxhyLywXPfwv2bQDEO/uMG2CBbowxXW7T806YX7kQJl7n62qOyfrQjTGmLfXV8O5vYdAUGH+Nr6tpl12hG2NMWz55GCr3wTVPQkj3v/61QDfGBJb6GqivOvH9VJfCJ4/AuG9C8qknvr8uYIFujAkc296Cl78HDdXe2V9oJMz8rXf21QUs0I0xgeGLR+Htn8HASTD5eu/sc9AUiE/2zr66gAW6McY78jNg/9Yjy8mnQUKqZ99bVQLZK0Bdx/fZ+zbAmsdg5EVw9eMQEXN8+/FzFujGmBO3+p+w7E5Aj6wLj4Frn4a0Gcf+3v1b4ZmroSL/xGpIvwUuuh9CgzfWgvfIjTEnzuWClb+Cz/4Goy6BWX8ACXVuSr4yD569Fi57CKbc2Pr37/oIFl8P4VFw89Lj794IjYC4/sd/HAHCAt0Yc3waauG1WyHzVZg6D2bdCyGhR7Z/eym8cBMs+SFkvgYRPY7+flWnm6V3Ctzwkl/1VR+v2oYmSirr6BkVTq8e4V7fvwW6Mabjag44V9Z7P4WZv4PTfwjS4rX4yDiY+zysvBty3299P6MugssehujenV5yRx2sqWdnSTWlVXWUVdVTU99IVHgoMZGhuFyQX36YvQdqKK2qIzIshJjIMKLCQxD3z6GpSTlQU09ZVR1l1fWUVtZRXe+MBfOHK8czd5r3/wdmgW6M6ZjyPbDoaijfDVc/ASdd1Xbb0HCY9ccuK609tQ1NXwV0XaNzA1ZVqal31pdW1ZOzv4r1eeXklrT/6OOAnlH06xlJfaOL6vpGahtcqPs2QohAn5gI+sZGMLF3PAmxkSTERZAQE8kpKX065fg8CnQRmQU8DIQCj6nqvS22JwNPAfHuNneq6lIv12qM8YWKffDUpXAwz1l2NThX3996DYad4dvaWlHf6CK3tIrtRZVsL6pkT1kNew84fw4dbmj3+xNiI5iU1JurpgxhzMA4EmOjSIiLoEdEGLUNTdTUN6GqDIqPJio8tN39daV2A11EQoEFwEwgH1gjIktUNatZs18CL6jqP0RkLLAUGNYJ9Rpjutq79zjDx067FSTE6SefOAcS0rq0DFWlorYRdV8C1za42FJwiHV7y9lccIiiQ7WUVddTXlP/1VVyeKiQ1LsHSX16MDGpFwN7RZMQG0HfmEiiI46EcXREKImxkfSNdYK7Lb2ivd/v7U2eXKFPBXJUNRdARBYDVwDNA12Bnu6vewGF3izSGOMjheth47Nwxm0w8x6v7rqhyUXRodqvlkNChB7hofSIDKWu0cXGvIOs23OQLYWHyHNfYdfUf3088rAQYfTAOEYkxjI1JYKE2EiGJ8YwekBPUhJiiAjr/mOweIsngT4YyGu2nA9Ma9HmN8AKEfkhEAO0+uCpiMwD5gEkJwf+HW1j/JoqLPsF9EiAs/7HK7t0uZR1e8t5fUMhb23ex4Hq+mO2F4HhCTGkJMRw2oi+DOoVTVioc9PRCfKenDSo11FX28HMk0BvbUR3bbE8B3hSVf8sIqcB/xGRk1SPfu1LVRcCCwHS09Nb7sMY051kve48xXLpXyCqZ/vt27Dv0GE+2lHKxzmlfLqzjNKqOqLCQ5gxpj9npiYQGuJ+KsSlHHb3UYvAhMHxTEjqRc+o7t3N0Z14Euj5QFKz5SF8vUvlFmAWgKp+JiJRQAKw3xtFGmO86MAuWPskuBqP3S7zNeg3Dia38VLQMewormTZliJWZhWzueAQAIlxkZyZ2pdzRiUyc+wAYiPtITtv8+QnugZIE5EUoACYDcxt0WYvMB14UkTGAFFAiTcLNcZ4QX6G8/Zm7SEIizp224hYuOQBj1+lzztQwxubClmyoZBtRZWIwKSkeP531mjOH92Pkf1jv3pG23SOds+UqjaKyHxgOc4jiU+oaqaI3ANkqOoS4CfAoyJyB053zM365a1oY0z3sO0teOkW5xX576zwfOCsVqgqZdX17C6t5sPsUlZmFbN1XwUAU5Lj+e3l47ho/AD6xbXzPw3jVeKr3E1PT9eMjAyffLYxAcnlgvf/6LyK/zUKB3KdoWXnvgCxiR7vVlXZXVZDxu4DrM87yIa9B9ldVv3VEychAicP7c3Msf2ZNW4gyX17tLNHcyJEZK2qpre2zTqxjAkEDbXw2vch8xUYfi5Et/Im4shZcN4v2h1atqa+kdLKevIP1vD+9hJWZhWzq9R5azIuKoxJSfGcOrwvyX2iSerTg0lJ8fSNjfT+MZkOs0A3xh81HHbmugRoaoA374A9nzjPip/+o6+Pq3IM24oq+DjbeQpl7e5yKuuO3CwNDxVOG5HAd84Yxmkj+jI8IZaQEOsH764s0I3xR//5pvNI4ZdCI+Cqx2H81R7vorqukV+9toVX1hcAMCIxhssnDWJI7x70jY2gX1wkJw/tTZw9Nug3LNCN8Tf7tzlhPuVGGOoeS2XABOg/1uNdbC+q5AeL1pJbWs2Pzk9l7rShDOhlNzD9nQW6Mf5mwyIICYPz7+7QzU2AxiYXT366mwdWbCcuKpxF353G6SMSOqlQ09Us0I3xJ02NsOl5SLuww2G+bm85d726ha37Kjh/dD/uu2oCiXF2MzOQWKAb409y3oGq4g7Nar+9qJIFq3J4Y1Mh/eOi+OcNU7hw3AB7yScAWaAb4082PAMxiZB2QbtNs4sreWDFdpZnFhMTEcp/nT2C+een2iv3AczOrDH+oroMti+Daf/lzATUhsYmF//6MJeH38kmMjyEH01P49unD6N3TEQXFmt8wQLdmO7A5eLrg5i2sOl5Z7agSS2HUnKoKhl7yvntG5lsKajgkgkDuefycfbSTxCxQDfG12or4G/pTt94ewZOgv7jjlp1sKaeV9YV8NwXe8neX0VCbAT/uH4KF40f2EkFm+7KAt0YX9v1gRPmp3wXYvsfu+3IWV99mbO/iic/3cXLaws43NDEpKR47r9qApdMGEiM9ZMHJTvrxvha9kqIiINZ9x6zb/xL6/aWs+C9HN7dtp+IsBC+MWkQN5+ewthBxz8JhQkMFujG+JKq8yji8HPaDfO1ew7wl5XZfJxTSnyPcG6fkcYNpw4lwfrIjZsFujG+VLINKgrgnJ+12aSusYkHV+xg4Ue59I2J5BcXj+b6aUOtW8V8jf1GGONL2Sudv1Nntrp5e1Eldzy/gax9FcydlswvLxlDjwj7z9a0zn4zjPGlnJXQbyz0GvzVqvpGF+9sLea5L/bycU4pfXpE8NiN6cwY284NUxP0LNCN8ZW6KtjzGZx6K+AMZ/vs53tZ+FEuJZV1DOoVxW3TrZ/ceM6jQBeRWcDDOHOKPqaq97bY/hfgPPdiD6CfqsZ7s1BjAs6uD8HVQMmAs3n2nWye/HQX5TUNnJHal/uuGs85I/sRapNJmA5oN9BFJBRYAMwE8oE1IrJEVbO+bKOqdzRr/0NgcifUakzAOHS4geJPXiFJojn92cM0sIPzR/dj/vmpTEnu7evyjJ/y5Ap9KpCjqrkAIrIYuALIaqP9HODX3inPmMBR29DE9qxNrNy4iw+z9/P3kFWsC5/A7ReO47IJg2xyZXPCPAn0wUBes+V8YFprDUVkKJACvHfipRnj31wu5YMdJbyyvoCswkP0PbCeF8J/w0Tgf0KdNoNn/ZIz0lN9WqcJHJ4EemudeG2NIjQbeElVm1rdkcg8YB5AcnKyRwUa429qG5p4aW0+T3yyi9ySahJiI5mcHM9tUWtoKI+h6bK/ERUeBqERyIjzfV2uCSCeBHo+kNRseQhQ2Ebb2cB/t7UjVV0ILARIT09vZ2g5Y/xLVV0jz6zew2Mf7aK0qo4JQ3rx8OxJXDx+IOGN1fDAKphwNeETv+nrUk2A8iTQ1wBpIpICFOCE9tfG7xSRUUBv4DOvVmhMN1ZaVccnOaV8mlPG8qwiDtY0cFZaAvPPm8zUlD5HZgXa9Do01MCkG3xbsAlo7Qa6qjaKyHxgOc5ji0+oaqaI3ANkqOoSd9M5wGJVtStvE9BKq+p4a9M+lmwsZO2ecgB6RoVx1shEvnfWcCYltfLE7vpF0DcNkqZ2cbUmmHj0HLqqLgWWtlh3d4vl33ivLGO6n9KqOv64dBuvbSigyaWMHhDHT2aO5OyRiZw0uFfbz4yX7YS9n8L0X4PN42k6kb0pakw7XC7l+Yw87n17GzX1jdx02jCuPWUIowd4OFzthmdBQmDinM4t1AQ9C3Rj2lBeXc8r6wtY7J4JaFpKH35/5Umk9ovzfCeuJtj4HIyYDj1tBiHTuSzQjWlhZ0kV/3h/J0s2FlLf6GJSUjwPz57E5RMHHbnJ6QlV+OhBZ3jcC3/feQUb42aBbgzOBMtbCir414c7eWvzPiLDQrguPYk5U5OPbyYgVxMs/SlkPA4nXQ2jL/N+0ca0YIFugtru0mqWbCzk9Q0F7CypJiYilFvPGcEtZ6Z0bITD2grY+a4T5ACbX4Idb8MZtzs3Q0NCOucAjGnGAt0Enc35h3h7yz5WZhWTvb8KgKkpffjOmSlcOn4QvXq0P6/nUQ7mwaJroGTrkXUSAhc/AFO/58XKjTk2C3QTFBqbXCzPLObxj3NZt/cgoSHC1GF9mDM1mQtPGsDg+Ojj23HRZifM66th9rOQMNJZH9ULYvt57wCM8YAFuglY9Y0uPt9VxsqsYlZkFlNUUUtynx78+rKxXDl5MPE9IiDjCXj3eF9uVti+DKJ6wneWQf9xXq3fmI6yQDcBZ1tRBc99vpdX1xdQUdtIVHgIZ6Ulcs/J45g+pv+RF4D2roY374DYARB+nFfoQ06GK/5+1BRyxviKBbrxa/sOHSZjdzl7yqrZe6CGrfsq2VxwiIjQEGadNIDLJg7izNQEoiNCj/5GlwuW/RziBsIP10JEjG8OwBgvskA3fuVQTQOf5Zbx6c5SPs4pJbek+qtt/eIiGdq3B7+6dCzfnDyY3jERbe9o84tQuA6+8U8LcxMwLNCNX1iRWcSC93eyOf8gLoXo8FCmDe/DnFOSOW1EX0Ykxn79Krwt9dXwzm9g0GSYcF2n1m1MV7JAN93agep6fr0kkzc2FpLWL5b556dxZmoCk5LiiQhr5dnusp1QXXrsnWa+CpWFcPXj9ny4CSgW6KbbUVW2FVWyMquYpz7dTUVtAz+eOZLvnzuC8NBjBHD2O7DoatqeUKuZcVfC0NO9VrMx3YEFuukWGptcfLH7ACuzinlnazF5Bw4jAlOH9eG3V4xrf2TDpgZY/nPokwIX/4nWZ050CwmFoWd4tX5jugMLdNPlVJVdpdVsKaxge1EF24sqWbO7nEOHG4gIC+HM1AR+cG4q08f0o19clGc7zfg3lO6A2c9B6ozOPQBjuikLdNMlDtc38eLaPN7btp8NeQc5WNMAQGiIMDwhhhlj+jNzbD/OSkskJrKDv5aHy+H9P0DK2TDqok6o3hj/YIFuOtWRiZNzKa2qZ0RiDBeOHcDk5HgmDIlnRL8YIsM8fDqlLR/8CQ4fhAv/YDMCmaDmUaCLyCzgYZw5RR9T1XtbaXMt8BucO1IbVfVrE0mb4LB1XwXvbdvPx9mlrN1bTn2jyz1xcirThvf17oeV7YQv/gVTboQB4727b2P8TLuBLiKhwAJgJpAPrBGRJaqa1axNGvBz4AxVLRcRG5UoCGXsPsBf38vhgx0lAIwZ2JMbTx3KpRMHtT5xsjes+CWERcP5v+yc/RvjRzy5Qp8K5KhqLoCILAauALKatfkesEBVywFUdb+3CzXdk6ry2c4yHnkvm9W5B+gbE8HPZo3i2vSkjo0nfjxy34ftS53xxm1kQ2M8CvTBQF6z5XxgWos2IwFE5BOcbpnfqOoyr1RouqUml/L+9v0sWJXDur0H6d8zkl9dOpa5U5M9f2PzRLiaYPldEJ8Mp/6g8z/PGD/gSaC3dpep5ZsbYUAacC4wBPhIRE5S1YNH7UhkHjAPIDk5ucPFGt8rOlTLCxl5PL8mj4KDhxkcH83/feMkrj55CFHhXRDkX1r/HyjeAtc8CeEePtpoTIDzJNDzgaRmy0OAwlbarFbVBmCXiGzHCfg1zRup6kJgIUB6eroHr/OZ7qK+0cWCVTksWJVDo0s5MzWBn188mgvHDTj225vecvggbHsLXA3O5Murfg/Jp8HYb3T+ZxvjJzwJ9DVAmoikAAXAbKDlEyyvAXOAJ0UkAacLJtebhRrfcLmULYWH+NlLm9hWVMkVkwbx45kjGdq3C0coLN/jvNJfuuPIuvAYe0zRmBbaDXRVbRSR+cBynP7xJ1Q1U0TuATJUdYl72wUikgU0AT9V1bLOLNx4X0VtA+9vL+GdrGJW55ZRWdvI4QZn0uN+cZE8emM6M8f279qiCjfAs9dCYy3c8DL0G+usj4h1ZgoyxnxFVH3T85Genq4ZGRk++WzjcLmU9XnlfJxdxic7S1m3p5xGl9I3JoKzRyaSEBtBdEQY8dHhXDVlSMcnTz5R2e/ACzdCjz5w/UvQb3TXfr4x3ZCIrFXV9Na22ZuiQaixycUbmwpZsGonOfurEIFxg3pyy1kpzBzTn8nJvY9M0+Yr656GN26H/mNh7ovQc6Bv6zHGD1igB5Ga+kZeWVfAox/lsqeshlH943jw2omcN6rfsWf36Uqq8P4f4YP7YMR0uPYpiIzzdVXG+AUL9CCwv7KWf3+ym2c/38uhww1MHNKLu751MjPG9CfEF1fitYfg5e/Bzne/vk0VtAkm3QCXPQShXdzNY4wfs0APYLUNTTz+8S7+viqHww1NXDhuALecmcLJQ3sjvno6pKIQFl0DJdvglO+2Pp9nnxEwaa49wWJMB1mgB6gPd5Twi1c3k19+mAvG9ufnF48hJcHLjxoWbYHqDozyUF8Nb/8v1FbA9S/CiPO9W48xQc4CPQC9si6fn760ieEJMTz73Wmcnprg3Q9QhQ/ud8Yg76i4gfCdt21kRGM6gQV6gHni413c82YWZ6T25V/fSie2o5NFtKepAd68w3n1fsJsSP92x74/cTREd9LIi8YEOQv0AKGqPLhyB399L4dZ4wbw8JxJHZs4onw3rHncCexjKdoEez6Bs38K591l/dzGdCMW6AGgocnFnS9v5uV1+VyXnsTvrzyJsI6Mr1KwFp69zhkvJbzHsduGRcBlD8PJN59QzcYY77NA93OVtQ38YNE6Psou5fYZadw2Pa1jT7BsXwYvfRtiEuHbb0NCWucVa4zpVBbofqq+0cVr6wv426ocCg8e5v6rJ3BtetKxv0kVPv0rrHsK1OWsK98NAyfC3Bdskghj/JwFuh96MSOPh97JpuDgYcYP7sV9V03gtBHtzNXZ1Ahv/xQynoDk06HXYGf96Evg3J+3/jy4McavWKD7i9oKXJXF/P39nby8Lp9JA+N4cPowpqb0QeJbeTW+qcEZdhacq/GVd8OOt+GM250p20K6YAxzY0yXskD3B1Ul6N9PJaSmlPnA/EjgAPCWe3viaGc0wnh3l8vBPPfbmFuP7ENC4OIHYOr3urZ2Y0yXsUD3AyVv3E3vmnJ+1fBdLpg0nPNGNevrrjsE79wDj81w3r4EZ/zw+mq45M8Q2ctZl5AKgyZ3ffHGmC5jgd6N7Siu5Nk33uZXec/xQshFnD37fzhvfCvDyCaf7szo8++LAHEmfvjOMug/rstrNsb4jgV6N1RaVcefV+zg+TV7WBTxCA3hcVw+/xFi4hNb/4b+Y+G77zjPkqMw5/kjNz2NMUHDAr0bqW1o4unPdvPXd53REf9vbCGn7dwMM+6DtsL8Sz0HwbwPnK/thqcxQckCvRtobHLxyroCHnpnB4WHajlvVCJ3zUoj9aWZ0DcNTrnFsx1ZkBsT1DwKdBGZBTyMM0n0Y6p6b4vtNwN/Agrcq/6mqo95sc6AtTq3jLte3czOkmomJsXzp2smckZqAqz+B5RlO90nNsmDMcYD7Qa6iIQCC4CZQD6wRkSWqGpWi6bPq+r8TqgxIFXXNXLfsm08/dkekvv04J83nMyF4/o7r+3XHID374Xh58HIC31dqjHGT3hyhT4VyFHVXAARWQxcAbQMdOOh7OJKvv3kGgoOHubbZwzjpxeOokdEs1PxwX1QVwEX/sFGMzTGeMyTQB8M5DVbzgemtdLuKhE5G9gB3KGqeS0biMg8YB5AcnJyx6sNAHvLarj+sc9R4MX/Oo30YX2OblCyA754FKbc5Dy9YowxHvIk0Fu7RNQWy28Az6lqnYjcCjwFfG1+MVVdCCwESE9Pb7mPgFd0qJa5j62msbGBJZc0MeTwJ7C1RaMvHnXGVTnvLp/UaIzxX54Eej7QfBi/IUBh8waqWtZs8VHgvhMvLbCUVtVx/WOrqa2pYtXQp+n15sq2G1/4R4ht5zFFY4xpwZNAXwOkiUgKzlMss4G5zRuIyEBV3edevJyvX3cGteKKWuY+uprDB4t4v//fic3b7PSPp5z99cZh0c5r+sYY00HtBrqqNorIfGA5zmOLT6hqpojcA2So6hLgRyJyOdCIM2zUzZ1Ys/84sIvqVQ+SsbWQ7zc0cWmvHKIOlsJ1zzjD1hpjjBeJqm+6stPT0zUjI8Mnn91Vql/6AZFbnqOMeHr3iCAitg9c9ggkneLr0owxfkpE1qpqemvb7E3RTqJ1VUjmq7yhZ5P6vafpP6SXr0syxgQ4e1e8k3y+9Cl6aA2xp97EeAtzY0wXsEDvBEWHapENiygKHcj0C77h63KMMUHCAt3LVJWHXlzJNMkkMv1bhITaj9gY0zUsbbxIVXn8410M2P0KitD7tBt9XZIxJojYTVEvqaxt4M5XNrN0UwFrYj5Bk89B4pPa/0ZjjPESC3Qv2FFcyfeeziC//DAPTasmYWMxTP69r8syxgQZ63I5QbUNTdz6zFqq65pYPO9UrtBVENkTxlzq69KMMUHGAv0EPbhyB7kl1Tx03SROGRAGWa/DSVdBeLSvSzPGBBkL9BOwdk85j36Uy9xpyZyZlgCZr0LjYZh8g69LM8YEIQv041Tb0MRPX9zIoF7R/OLiMc7KDYsgYRQMPtm3xRljgpIF+nF66J1sckuruf/qCcRGhkFpNuR9DpPm2ixDxhifsEA/DrklVTz+cS7Xpg9xJnQG5+pcQmHibN8WZ4wJWhbox+F3b2YRFRbKz2aNdla4mmDjYkidAXEDfFucMSZoWaB30Hvbilm1vYTbZqSREBvprNy5Cir3weTrfVucMSaoWaB3QH2ji9+9uZXhiTHceNqwIxs2PAPRfWDkRT6rzRhjLNA74N+f7GJXaTV3XzqWiDD3j67mAGx7CyZcC2ERvi3QGBPULNA9VFxRyyPvZnP+6H6cO6rfkQ1bXoamephk3S3GGN/yKNBFZJaIbBeRHBG58xjtrhYRFZFWp0fyZ79/aysNLuXXl409esP6Z2DAeBg4wTeFGWOMW7uBLiKhwALgImAsMEdExrbSLg74EfC5t4v0tc92lrFkYyG3nj2coX1jjmwozoR9G2CSvRlqjPE9T67QpwI5qpqrqvXAYuCKVtr9DrgfqPVifT7X0OTi10u2MDg+mu+fm3r0xvWLICQcxl/jm+KMMaYZTwJ9MJDXbDnfve4rIjIZSFLVN4+1IxGZJyIZIpJRUlLS4WJ94alPd7OjuIq7LxtLdETokQ1NDbDpeRg1C2L6+q5AY4xx8yTQW3uPXb/aKBIC/AX4SXs7UtWFqpququmJiYmeV+kjB6rrefjdbM4emcgFY/sfvXHHcqgpte4WY0y34Umg5wPNp94ZAhQ2W44DTgLeF5HdwKnAkkC4MfrIu9lU1zXyq0vGIM3HZ1GF1f+A2P7O26HGGNMNeBLoa4A0EUkRkQhgNrDky42qekhVE1R1mKoOA1YDl6tqRqdU3EV2lVbzzOo9XHdKMmn9447euO1N2PMxnPMzCLVJn4wx3UO7ga6qjcB8YDmwFXhBVTNF5B4RubyzC/SV+97eRkRYCHfMTDt6Q2MdrPgVJI6BKTf7pDZjjGmNR5eXqroUWNpi3d1ttD33xMvyrTW7D7Ass4gfzxxJv7ioozd+sRDKd8ENr9jVuTGmW7E3RVtQVX7/1lb694zku2elHL2xuhQ++BOkXQCp031ToDHGtMEuMVt4a/M+Lt/3CN/svZMej7UYm6X2ENRXwQX/55vijDHmGCzQm6lrbOIvb29kedhKQqNHQZ+UrzcafRckjur64owxph0W6M3857M99D2YSVhkE0y/G0bZcLjGGP9hfehuB2vq+et7OVzdz/2I/ZCpvi3IGGM6yK7Q3f72Xg6VtQ1clLwHwlLtdX5jjN+xK3Rgd2k1T322m6unDCZu/zpIOtXXJRljTIdZoAN/fHsr4aEh/O/UcDh8AJKsu8UY43+CPtA/21nG8sxifnDuCPoe2OCsTJrm26KMMeY4BHWgN7mU372ZxeD4aL571nDI+xyiekHCSF+XZowxHRbUgf7y2nyy9lXws1mjiAoPhbwvnKdbQoL6x2KM8VNBm1x1jU08sGI7k5PjuXziIDhcDiVbrbvFGOO3gjbQV2YVs7+yjttnjHTGOs93j/ZrN0SNMX4qaAP9+TV5DI6P5qzUBGdF3ucgoTD4ZN8WZowxxykoAz3vQA0fZZdybXoSISHumYjyPocBJ0FkrG+LM8aY4xSUgfNgj3AAAAweSURBVP5CRh4hAtekD3FW1FVC3hp7ocgY49eCLtAbm1y8mJHPOSMTGRQf7azMfA0aD8NJV/m2OGOMOQFBF+gfZpdQVFHLdackH1m5YRH0TbMbosYYv+ZRoIvILBHZLiI5InJnK9tvFZHNIrJBRD4WkbHeL9U7Fn+RR0JsBNPH9HNWlO2EvZ/BpLkg4tvijDHmBLQb6CISCiwALgLGAnNaCexnVXW8qk4C7gce9HqlXlBaVce72/Zz1ZQhhIe6D33DIpAQmDjHt8UZY8wJ8uQKfSqQo6q5qloPLAauaN5AVSuaLcYA6r0SvWfZliKaXMqVUwY7K1xNsOE5GDEdeg70bXHGGHOCPAn0wUBes+V897qjiMh/i8hOnCv0H7W2IxGZJyIZIpJRUlJyPPWekKWb9zE8MYZR/eOcFbnvQ2UhTL6+y2sxxhhv8yTQW+tY/toVuKouUNURwP8Cv2xtR6q6UFXTVTU9MTGxY5WeoNKqOlbnlnHp+IHOm6HgdLdE94ZRF3dpLcYY0xk8CfR8IKnZ8hCg8BjtFwPfOJGiOsOyLUW4FC6e4O5aOZQPW9+E8ddAWKRvizPGGC/wJNDXAGkikiIiEcBsYEnzBiKS1mzxEiDbeyV6x1ub9jGieXfLO79xnmo5/Yc+rcsYY7yl3TlFVbVRROYDy4FQ4AlVzRSRe4AMVV0CzBeRGUADUA7c1JlFd1RJZR2f7ypj/nmpRwbi2vwinPUTiE9ufwfGGOMHPJokWlWXAktbrLu72de3ebkur1qW6XS3XDJhEKjCsjshtj+ceYevSzPGGK/xKND93VubChmRGMPI/rGw5WXIXwOX/w0i43xdmjHGeE3Av/q/v7KWL3Yd4JLxA5GmBqfvfMB4581QY4wJIAF/hf7S2nxcCldMHgw73oZDeXDJnyEk1NelGWOMVwX0FbrLpSz+Io9pKX0YkRgL6xdB3CBIneHr0owxxusCOtA/3VnG3gM1zJ2WDJVFkLMSJs62q3NjTEAK6EB/7ou99O4RzoXjBsDGxaAumGSv+RtjAlPABnpJZR3LM4u4asoQosJCnNf8k6ZBQqqvSzPGmE4RsIH+8rp8Gl3K7KnJzotEpTvs6twYE9ACMtCdm6F7mZrSh9R+sbDhGQiLhnFX+ro0Y4zpNAEZ6Bl7ytldVsOcqUlQcwC2vAJjr4Conr4uzRhjOk1ABvryzCIiwkK4YFAdPH4BNNbCtHm+LssYYzpVwL1YpKqsyCrihqQyYp6eBU31cOPrMPhkX5dmjDGdKuACfVtRJZTv5ud1v4DYRLj5TUgc5euyjDGm0wVcoK/MKubi0C8IbzoMN70OfYb7uiRjjOkSAdeHviKriEuis6DfOAtzY0xQCahALzh4mF0FxYxrzITU6b4uxxhjulRABfo7WcWcHpJJqDZC2kxfl2OMMV0qoAJ9RVYRl8VkQUQsJJ3q63KMMaZLeRToIjJLRLaLSI6I3NnK9h+LSJaIbBKRd0VkqPdLPbZDNQ18nlvGObIRUs6BsIiuLsEYY3yq3UAXkVBgAXARMBaYIyJjWzRbD6Sr6gTgJeB+bxfanlXb9zNUC+hVvw/SbLxzY0zw8eQKfSqQo6q5qloPLAauaN5AVVepao17cTUwxLtltm9FVhGXRmc6CzaBhTEmCHkS6IOBvGbL+e51bbkFeLu1DSIyT0QyRCSjpKTE8yrbUdvQxAfbS7i0RyYkjIL4ZK/t2xhj/IUngS6trNNWG4rcAKQDf2ptu6ouVNV0VU1PTEz0vMp2fLazDFd9NSNqNtjVuTEmaHnypmg+kNRseQhQ2LKRiMwA7gLOUdU675TnmRVZxZwXuYMQV4P1nxtjgpYnV+hrgDQRSRGRCGA2sKR5AxGZDPwLuFxV93u/zLa5XMrKrGKui98G4T0g+fSu/HhjjOk22g10VW0E5gPLga3AC6qaKSL3iMjl7mZ/AmKBF0Vkg4gsaWN3Xrc+7yClVbWkN6yFYWdBeFRXfbQxxnQrHg3OpapLgaUt1t3d7Guf9XOszCpmREgxMTV5kHabr8owxhif8/s3RVdkFXFzv2xnwW6IGmOCmF8Hes7+KnJLqjkvbDP0GQF9UnxdkjHG+IxfB/rbm/cRST2DyzNsMC5jTNDz20BXVV7dUMBNA/ORplpItUA3xgQ3vw30TfmHyC2p5qpe2yAsCoad4euSjDHGp/w20F9dX0BEWAipFZ/BsDMhPNrXJRljjE/5ZaA3NLl4Y2Mh16U2Enpgpz3dYowx+Gmgf5xdSll1PTf3+MxZkXaBbwsyxphuwC8D/dX1BYyMrmD4jidg3JXQd4SvSzLGGJ/z6E3R7qSqrpEVWUU8l/AqUuGCGb/1dUnGGNMt+N0V+vItRYxszGZy+XI47QfQu8tnuzPGmG7J767Q+8SE85dez6Oh/ZAzf+zrcowxptvwu0A/r/ETqN0Clz0CUT19XY4xxnQbftflQmQcjL4UJt/g60qMMaZb8bsrdNJm2rgtxhjTCv+7QjfGGNMqC3RjjAkQFujGGBMgPAp0EZklIttFJEdE7mxl+9kisk5EGkXkau+XaYwxpj3tBrqIhAILgIuAscAcERnbotle4GbgWW8XaIwxxjOePOUyFchR1VwAEVkMXAFkfdlAVXe7t7k6oUZjjDEe8KTLZTCQ12w5372uw0RknohkiEhGSUnJ8ezCGGNMGzwJdGllnR7Ph6nqQlVNV9X0xMTE49mFMcaYNnjS5ZIPJDVbHgIUnugHr127tlRE9hzntycApSdagx8KxuMOxmOG4DzuYDxm6PhxtzkioSeBvgZIE5EUoACYDcztwIe3SlWP+xJdRDJUNf1Ea/A3wXjcwXjMEJzHHYzHDN497na7XFS1EZgPLAe2Ai+oaqaI3CMil7sLOkVE8oFrgH+JSKY3ijPGGOM5j8ZyUdWlwNIW6+5u9vUanK4YY4wxPuKvb4ou9HUBPhKMxx2MxwzBedzBeMzgxeMW1eN6YMUYY0w3469X6MYYY1qwQDfGmADhd4He3kBhgUBEkkRklYhsFZFMEbnNvb6PiKwUkWz33719Xau3iUioiKwXkTfdyyki8rn7mJ8XkQhf1+htIhIvIi+JyDb3OT8tSM71He7f7y0i8pyIRAXa+RaRJ0Rkv4hsabau1XMrjkfc2bZJRKZ09PP8KtA9HCgsEDQCP1HVMcCpwH+7j/NO4F1VTQPedS8HmttwHo/90n3AX9zHXA7c4pOqOtfDwDJVHQ1MxDn+gD7XIjIY+BGQrqonAaE477gE2vl+EpjVYl1b5/YiIM39Zx7wj45+mF8FOs0GClPVeuDLgcICiqruU9V17q8rcf4DH4xzrE+5mz0FfMM3FXYOERkCXAI85l4W4HzgJXeTQDzmnsDZwOMAqlqvqgcJ8HPtFgZEi0gY0APYR4Cdb1X9EDjQYnVb5/YK4Gl1rAbiRWRgRz7P3wLdawOF+QsRGQZMBj4H+qvqPnBCH+jnu8o6xUPAz4AvR+3sCxx0v9wGgXm+hwMlwL/dXU2PiUgMAX6uVbUAeABn6O19wCFgLYF/vqHtc3vC+eZvge61gcL8gYjEAi8Dt6tqha/r6UwicimwX1XXNl/dStNAO99hwBTgH6o6GagmwLpXWuPuN74CSAEGATE4XQ4tBdr5PpYT/n33t0DvlIHCuiMRCccJ80Wq+op7dfGX/wRz/73fV/V1gjOAy0VkN05X2vk4V+zx7n+SQ2Ce73wgX1U/dy+/hBPwgXyuAWYAu1S1RFUbgFeA0wn88w1tn9sTzjd/C/SvBgpz3/2eDSzxcU1e5+47fhzYqqoPNtu0BLjJ/fVNwOtdXVtnUdWfq+oQVR2Gc17fU9XrgVXAl9MaBtQxA6hqEZAnIqPcq6bjTB4TsOfabS9wqoj0cP++f3ncAX2+3do6t0uAG91Pu5wKHPqya8ZjqupXf4CLgR3ATuAuX9fTScd4Js4/tTYBG9x/LsbpU34XyHb/3cfXtXbS8Z8LvOn+ejjwBZADvAhE+rq+TjjeSUCG+3y/BvQOhnMN/BbYBmwB/gNEBtr5Bp7DuUfQgHMFfktb5xany2WBO9s24zwB1KHPs1f/jTEmQPhbl4sxxpg2WKAbY0yAsEA3xpgAYYFujDEBwgLdGGMChAW6McYECAt0Y4wJEP8PrUXrmp9v/vAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores1)\n",
    "plt.plot(test_scores1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# absolute value rectification + Sigmoid + Cross Entropy  \n",
    "没有在pytorch中找到absolute value rectification的接口"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Leaky ReLU + Sigmoid + Cross Entropy Cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net2(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net2, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(28*28, 30)\n",
    "        self.fc2 = nn.Linear(30, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.leaky_relu(self.fc1(x))\n",
    "        x = F.logsigmoid(self.fc2(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net2 = Net2()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net2.parameters(), lr = 1e-1)\n",
    "loss_scores2, train_scores2, test_scores2 = fit(net2, criterion, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXhV5b328e8vOxOBkDBkYDQMYZKZMIqC4gCiglOplQqIB1GPY3tq2/e1rW1tzzmtY1UoVUBxrlIHtKCiAjIngMxzGMKUQBgCgYzP+0eiF68mJJCdrOyd+3NduWDvtdzrXi68XTz7WWuZcw4REQl8IV4HEBER/1Chi4gECRW6iEiQUKGLiAQJFbqISJAI9WrDTZs2dUlJSV5tXkQkIKWlpR12zsWVtcyzQk9KSiI1NdWrzYuIBCQz213eMg25iIgECRW6iEiQUKGLiAQJFbqISJBQoYuIBAkVuohIkFChi4gEiYAr9PTDp3jy0y0s2JrFiTMFXscREak1PLuw6EKt33ecF77cTrEDM+iU2JA+F8WSclFj+lzUiJaN6mFmXscUEalx5tUDLlJSUtyFXil6Mq+QNXuOkbo7m7TdR1m1+yin8osASGgYQUpSY/pe1IiUpMZ0btYQX4gKXkSCg5mlOedSyloWcGfoAA0iQhmc3JTByU0BKCp2bD54grTdR0nddZS03Uf5eO0BAKIjQumT1Ih+bRrTv00TureMIcwXcCNNIiIVCsgz9MrYd+w0qbuyWZ6ezYr0bLZnngSgXpiPlKRGDGjbhIHtmtC9RQyhKngRCRDnOkMP2kL/vsMn81iZXlLwS3ccYcuhHKDkbL9/m8Zc0r7kjD85voHG4EWk1gq6IZcL0bRBBCO6NWNEt2YAHDmZx7Kd2SzZcZjF2w8zf3MmUDIGP7h9HJd1aMrg9k1p0iDCy9giIpVWZ87QK5JxNJfF2w+zcFtJwR/LLcAMujaP4bIOTRnSIZ5erWM1/i4intKQy3kqKnas33echVuzWLA1i9V7j1FU7IiOCGVQ+yYM6RDPkI5xtIit53VUEaljVOhVdPx0AUt3HGbB1iwWbMli//EzACTHN2BIhziGdoynb5tGRIT6PE4qIsFOhe5Hzjm2Z55kwdYsvtqSxYr0bPKLiokK9zGoXROGdoxnaMc4WjaK8jqqiAQhfSnqR2ZGckI0yQnR3HVpW3LzC1m64whfbcniyy2ZfL6p5MvVDgkNuLxjPEM7xpOS1Ehj7yJS7XSG7kfOOXZkneKrLZl8tSWL5elHKCgqGXsfnNy0pOA7xREfHel1VBEJUBpy8cjJvEIWbz/MV1sy+XJzFgdPlIy9d28Zw+Ud4xnWOZ6uzWMI0a0JRKSSqlToZtYKeBVIBIqBac65Z7+3zu3Ao6UvTwL3OOe+Odfn1oVCP5tzjk0HcvhySybzNx1i9d5jOAdx0RFc0TGeKzrHM7h9U+pHaBRMRMpX1UJvBjRzzq0ys2ggDRjtnNt41jqDgE3OuaNmNgL4nXOu/7k+t64V+vdln8pnwdZM5m/KZMHWLHLOFBLuC2FAuyZc2TmeYZ0TNC1SRH7Ar0MuZvYB8Lxz7rNyljcC1jvnWpzrc+p6oZ+toKiYlbuy+WJTJvM3Z5J++BQAnRKjuapLAld2TqBbCw3NiIgfC93MkoCFQFfn3Ily1vk50Mk5d1cZyyYBkwBat27dZ/fu3ZXedl2yI+sk8zcd4vONmaTuzqbYQXx0BMM6J3B1lwQGtmtCZJjmvIvURX4pdDNrACwAnnDOzS5nncuBF4HBzrkj5/o8naFXztFT+Xy1NZPPNh5iwZYsTuUXERXuY0iHOK6+OIErOiYQExXmdUwRqSFVLnQzCwPmAPOcc0+Vs0534F/ACOfc1oo+U4V+/vIKi1i64wifbTzEZxsPkZmTR2iI0b9tY665OJGruySSGKMpkSLBrKpfihrwCpDtnHuonHVaA18AdzjnllQmlAq9aoqLHWv3HWfehoPM23CQnVkl4+49W8Uyomsiw7smclGT+h6nFBF/q2qhDwYWAesombYI8GugNYBzbqqZvQTcDHw7KF5Y3ga/pUL3r+2ZOcxdf5B5Gw6xbt9xADo3a8iIrolc2y2R9vHRHicUEX/QhUV1zN7sXOZtOMjc9QdJ23MU50puJDaiWzNGdmtGhwQ9xEMkUKnQ67BDJ84wb8NBPll3gBXpJTNm2sXVZ2T35lzXvRkdEnTmLhJIVOgCQFZOHnM3HOTjtfu/K/f28Q0Y2a0Z1/dopmEZkQCgQpcfyMw5w7z1B/lo7QFW7srGuZILma7v0ZwbejSnVWPd/lekNlKhyzkdOnGGj9ce4KO1+1m95xgAvVrHckOP5ozs3kx3hxSpRVToUml7s3OZs/YAH6zZx+aDOYQYXNK+KTf0aM7wrolER+oiJhEvqdDlgmw7lMOH3+zngzX72ZOdS0RoCFd2TmB0rxYM6RBHeKge2iFS01ToUiXOOVbtOcaHa/YxZ+0BjpzKp1FUGNd1b86NvVvQq1WspkGK1BAVuvhNQVExX287zOzV+/h0w0HyCotp07Q+N/VqwY29W+hZqiLVTIUu1SLnTAH/Xn+Q2asyWLYzG4BB7Zpwc++WjOiWSFS4HtYh4m8qdKl2e7Nz+dfqfby3KoPdR3KpH+7j+h7NuTWlJb1bN9KQjIifqNClxjjnWLnrKO+m7WXO2gPk5hfRLq4+Y/q24qbeLWnaIMLriCIBTYUunjiVV8jHaw/wTupeUncfJTTEGNY5ntv6tebS5Dh8egKTyHlToYvntmfm8E5qBu+mZZB9Kp8WsfX4cd9WjOnbiviGunBJpLJU6FJr5BcW89nGQ7yxYjeLtx/BF2Jc1TmB2we05pJ2TfXcVJEKnKvQNQ1BalR4aAgjuzdjZPdm7Dp8ijdX7OGd1L3M3XCQNk3rc3v/1tzap5UeqydyAXSGLp7LKyzi3+sOMmvZbtJ2HyUyLITRPVswblASnZs19DqeSK2iIRcJGBv2H2fW0t28v2YfZwqK6ZfUmDsHJ3FVl0R9iSqCCl0C0LHcfN5J3curS3eTcfQ0LRvVY9zAJMb0a0VD3SBM6jAVugSsomLHZxsPMX1xOivSs2kQEcqYvq0YPyhJ92yXOkmFLkFh/b7jvLRoJ3PWHqDYOUZ2b87dl7Wla4sYr6OJ1JgqFbqZtQJeBRKBYmCac+7Z761jwLPAtUAuMN45t+pcn6tClwu1/9hpZi7ZxRvL93Ayr5BL2jfhniHtuaR9E91iQIJeVQu9GdDMObfKzKKBNGC0c27jWetcC9xPSaH3B551zvU/1+eq0KWqTpwp4I3le5j+dTqZOXl0bxnDvUPbcXWXRM1nl6B1rkKv8AkFzrkD355tO+dygE1Ai++tNgp41ZVYBsSW/o9ApNo0jAxj8pB2LHr0cv50YzeOny5g8muruOaZhXywZh9Fxd4MJ4p45bweOWNmSUAvYPn3FrUA9p71OoMflj5mNsnMUs0sNSsr6/ySipQjItTHT/q3Zv4jQ3j2xz0xgwffWsOVTy3g3bQMCouKvY4oUiMqXehm1gB4D3jIOXfi+4vL+Ed+cHrknJvmnEtxzqXExcWdX1KRCoT6QhjVswVzH7yMqWN7Uy/Mx8//+Q1XPb2Q2atU7BL8KlXoZhZGSZm/7pybXcYqGUCrs163BPZXPZ7I+QsJMYZ3bcbHDwxm2k/7UC/MxyPvfMPVzyxkztr9FGsoRoJUhYVeOoPlZWCTc+6pclb7ELjDSgwAjjvnDvgxp8h5MzOuvjiROfcPZurY3oSGGP/5xmpG/u1rvth8CK+m7IpUl8rMchkMLALWUTJtEeDXQGsA59zU0tJ/HhhOybTFCc65c05h0SwXqWlFxY6PvtnP059vZfeRXPolNebREZ3oc1Ejr6OJVJouLBI5S0FRMW+t3Muzn2/j8Mk8ru6SwC9HdKJtXAOvo4lUSIUuUobc/EJeXpTO1AU7yCss5vb+rXlgWDJN9Jg8qcWqNA9dJFhFhYdy/7BkvvqvyxnTtxWzlu1m6F+/4qVFO8kv1IwYCTwqdKnz4qIjeOLGbsx76DJ6tW7EHz/exPBnFvLllkyvo4mcFxW6SKnkhGhemdCX6eNL/jY7YcZK7nplJXuO5HqcTKRyVOgiZzEzruiUwNyHLuNXIzqxZMcRrnx6AU9/tpUzBUVexxM5JxW6SBnCQ0O4e0g7vvjZUK65OJFn529jxLOL+HrbYa+jiZRLhS5yDokxkfzttl7MmtgP5xxjX17Og2+t5vDJPK+jifyACl2kEi5NjmPuQ5fxwLBkPll3gKueWsDsVRm62lRqFRW6SCVFhvl45KoOfPzApSQ1rc8j73zD+Bkr2X/stNfRRAAVush565AQzbuTB/G767uwclc21zy9kLdX7tHZunhOhS5yAXwhxvhL2jD3wcvo0rwhj763jgkzV3LoxBmvo0kdpkIXqYLWTaJ48z8G8PgNF7Ns5xGueWYhn6zTjUbFGyp0kSoKCTHGDUri4wcu5aLGUdz7+ioeeWcNOWcKvI4mdYwKXcRP2sU14N17BvHAsGTeX72Pkc99zZq9x7yOJXWICl3Ej8J8ITxyVQfeuXsgRcWOW6YsYeqCHXpKktQIFbpINUhJaswnD1zKVV0S+O9/b+bOV1Zy9FS+17EkyKnQRapJTFQYL97emz+M7sqS7UcY+dwi0nYf9TqWBDEVukg1MjN+OuAi3rtnED6fMebvS5mxOF1z1qVaqNBFakC3ljHMuf9ShnaM5/GPNvKzd77hdL7u3ij+pUIXqSEx9cKY9tM+/OyqDvxrzT5unrKEvdm617r4T4WFbmbTzSzTzNaXszzGzD4ys2/MbIOZTfB/TJHgEBJi3D8smenj+pJxNJdRLyxm+c4jXseSIFGZM/SZwPBzLL8P2Oic6wEMBZ40s/CqRxMJXpd3iuf9+y4hNiqMsS8v560Ve7yOJEGgwkJ3zi0Ess+1ChBtZgY0KF230D/xRIJX27gG/OveSxjYrim/nL2O33+0kSLNV5cq8McY+vNAZ2A/sA540DlX5iPTzWySmaWaWWpWVpYfNi0S2GLqhTF9XAoTLkli+uJ0Jr+WRm6+zofkwvij0K8B1gDNgZ7A82bWsKwVnXPTnHMpzrmUuLg4P2xaJPCF+kL47fUX8/gNFzN/0yHG/H0Zmbpro1wAfxT6BGC2K7EdSAc6+eFzReqUcYOS+McdKezIOsmNLy5hR9ZJryNJgPFHoe8BhgGYWQLQEdjph88VqXOGdU7g7UkDySss4pYpS1i1R1eWSuVVZtrim8BSoKOZZZjZRDObbGaTS1f5AzDIzNYB84FHnXN6NLrIBerWMob37hlEw3ph/OQfy5i/6ZDXkSRAmFeXIKekpLjU1FRPti0SCA6fzGPCjJVsPHCCJ2/tweheLbyOJLWAmaU551LKWqYrRUVqqaYNInjjP/rTN6kRD7+zhllLd3kdSWo5FbpILRYdGcbMCf0Y1imexz7YwAtfbvc6ktRiKnSRWi4yzMeUsX0Y3bM5f5m3hSc/3aK7NUqZQr0OICIVC/OF8OSPehIR6uNvX2wnv7CYX47oRMkF2iIlVOgiAcIXYvz5pm6Eh4bw94U7yS8q5jfXdVGpy3dU6CIBJCTE+P2oiwnzhTB9cTqASl2+o0IXCTBmxmPXdcbhmLF4F0bJa5W6qNBFApCZ8ZvruuAcTF+cji8Efn2tSr2uU6GLBCgz47fXd6HYOf6xKJ3IMB8/u7qj17HEQyp0kQBmZvzu+ovJKyjmb19sJzLMx32Xt/c6lnhEhS4S4EJCjD/d1I0zhUX8Zd4W6oX5uHNwG69jiQdU6CJBwBdiPHlrD/IKivn9nI1ER4Zya0orr2NJDdOVoiJBItQXwrO39eTS5KY8+t5a5m046HUkqWEqdJEgEhHqY+rYPvRoFcv9b6xm8XbdybouUaGLBJn6EaHMGN+XNk3rM+nVVNZmHPM6ktQQFbpIEIqNCufVif2IjQpnwoyVpB8+5XUkqQEqdJEgldAwklkT++GAO6YvJzNHD54Odip0kSDWNq4B08f35XBOPuOnryTnTIHXkaQaqdBFglzPVrFMGdubrYdyuPf1VRQUFXsdSaqJCl2kDhjaMZ4/39SNRdsO86vZ6/SAjCBVYaGb2XQzyzSz9edYZ6iZrTGzDWa2wL8RRcQfbk1pxUNXJvNuWgZPf77N6zhSDSpzpehM4Hng1bIWmlks8CIw3Dm3x8zi/RdPRPzpwWHJ7D92mufmb6NlbD1+1FdXkwaTCgvdObfQzJLOscpPgNnOuT2l62f6J5qI+JuZ8cSN3Thw/Ay//tc6msfWY3ByU69jiZ/4Ywy9A9DIzL4yszQzu6O8Fc1skpmlmllqVlaWHzYtIucrzBfCC7f3pl1cA+55LY2th3K8jiR+4o9CDwX6ACOBa4DHzKxDWSs656Y551KccylxcXF+2LSIXIiGkWFMn9CXyHAfE2as1Bz1IOGPQs8A5jrnTjnnDgMLgR5++FwRqUYtYusxfVxfsk/lM+nVNM4UFHkdSarIH4X+AXCpmYWaWRTQH9jkh88VkWrWrWUMT4/pyTcZx/jZP7+huFjTGQNZZaYtvgksBTqaWYaZTTSzyWY2GcA5twmYC6wFVgAvOefKneIoIrXL8K6JPDq8Ex+vPcAzn2/1Oo5UQWVmudxWiXX+AvzFL4lEpMbdfVlbdmad5LkvttM2rgGje7XwOpJcAF0pKiKYGX8c3Y3+bRrzi/fWkrb7qNeR5AKo0EUEgPDQEKaO7UOzmEjunpVKxtFcryPJeVKhi8h3GtUP5+VxKeQVFHPXK6mczCv0OpKcBxW6iPx/2sdH8/ztJXdnfPjtNZr5EkBU6CLyA0M6xPHYdV34bOMhnvxsi9dxpJIqc3MuEamDxg9KYuuhk7zw5Q6S46M18yUA6AxdRMpkZjx+w8XfzXxZvUczX2o7FbqIlCs8NIQpY/uQ0DCCu2elcfC47vlSm6nQReScGtcP56U7+nIqr5BJs1J1z5daTIUuIhXqmBjNMz/uxbp9x/nFu2v1CLtaSoUuIpVyVZcEfn51Rz78Zj8vfrXD6zhSBs1yEZFKu3doO7YeyuGvn26hQ0I0V3VJ8DqSnEVn6CJSaWbG/9zcne4tYnjordVsPnjC60hyFhW6iJyXyDAff/9pCvUjQvmPV1PJPpXvdSQppUIXkfOWGBPJtDtSOHQij3teSyO/sNjrSIIKXUQuUM9Wsfzvzd1Znp7N7z7aoJkvtYC+FBWRCza6Vws2H8xh6oIddEqM5o6BSV5HqtN0hi4iVfJf13RkWKd4Hv9oI4u3H/Y6Tp2mQheRKvGFGM/8uCft4upz7+urSD98yutIdZYKXUSqLDoyjJfu6EuIwcRXVnL8dIHXkeqkCgvdzKabWaaZra9gvb5mVmRmt/gvnogEitZNopgytg97juRy/5urKSzSzJeaVpkz9JnA8HOtYGY+4H+AeX7IJCIBakDbJvxhdFcWbs3iT59s9jpOnVNhoTvnFgLZFax2P/AekOmPUCISuG7r15rxg5KYvjidN1fs8TpOnVLlMXQzawHcCEytehwRCQb/d2RnLusQx2Pvr2fpjiNex6kz/PGl6DPAo865Cm+SbGaTzCzVzFKzsrL8sGkRqY1CfSE8/5NeJDWtzz2vp7FLM19qhD8KPQV4y8x2AbcAL5rZ6LJWdM5Nc86lOOdS4uLi/LBpEamtGkaG8fK4FAy485WVHM/VzJfqVuVCd861cc4lOeeSgHeBe51z71c5mYgEvIua1Gfq2D7szc7l3jfSKNDMl2pVmWmLbwJLgY5mlmFmE81ssplNrv54IhLo+rdtwp9v6s7i7Uf4zQe650t1qvBeLs652yr7Yc658VVKIyJB6ZY+LdmZdZIXv9pBu7j63HVpW68jBSXdnEtEasTPr+5I+uFTPPHJJlo3juLqixO9jhR0dOm/iNSIkBDjqR/1pHuLGB58aw3rMo57HSnoqNBFpMbUC/fxj3EpNK4fzp2vrGTfsdNeRwoqKnQRqVHx0ZHMmNCXM/lFTJy5khNnNJ3RX1ToIlLjOiRE8+LY3mzPPMl9r6/SdEY/UaGLiCcuTY7jTzd1Y9G2w/x69jpNZ/QDzXIREc/8KKUVGUdP89z8bbRqHMUDw5K9jhTQVOgi4qmHr0wm42guT322lWYxkdya0srrSAFLhS4injIz/vum7mSeyONXs9cR3zCSIR10r6cLoTF0EfFceGgIU8b2pkNCNPe8lqY56hdIhS4itUJ0ZBgzJ/SlUVQ4E2auYPcR3XL3fKnQRaTWiG8YySt39qOw2HHH9BVk5eR5HSmgqNBFpFZpH9+AGeP7knkij/EzVpCjC48qTYUuIrVOr9aNeHFsb7YczOHuWWnkFVb4QDRBhS4itdTlHeP531u6s2THER58cw2Fupq0Qip0Eam1burdkt9c14W5Gw7y63/patKKaB66iNRqdw5uw7HTBTw3fxuxUeH8akQnzMzrWLWSCl1Ear2Hr0zmeG4+0xbupGFkKP95hW4RUBYVuojUembGb6+/mJwzhfz1061EhYdy5+A2XseqdVToIhIQQkKM/72lO7n5Rfx+zkbqR/gY07e117FqFX0pKiIBI9QXwrO39WRIhzh+OXsdH6zZ53WkWqXCQjez6WaWaWbry1l+u5mtLf1ZYmY9/B9TRKRERKiPqWP70C+pMY+88w2frDvgdaRaozJn6DOB4edYng4Mcc51B/4ATPNDLhGRctUL9zF9fF96torlgTdX8+mGg15HqhUqLHTn3EIg+xzLlzjnjpa+XAa09FM2EZFy1Y8IZcaEvlzcIob73ljFl5szvY7kOX+PoU8E/l3eQjObZGapZpaalZXl502LSF3TMDKMV+/sR6fEhtw9K63Ol7rfCt3MLqek0B8tbx3n3DTnXIpzLiUuTjewF5Gqi6kXxmsT+9MhsUGdL3W/FLqZdQdeAkY554744zNFRCorJiqM1ycO+K7Uv9h8yOtInqhyoZtZa2A28FPn3NaqRxIROX/flnrHxGjunpXG3PV174vSykxbfBNYCnQ0swwzm2hmk81scukqvwGaAC+a2RozS63GvCIi5YqJCuO1u/rTtfSL0jlr93sdqUaZV3cvS0lJcamp6n4R8b+TeYXcOWMlqbuz+eutPbipd/BMvjOzNOdcSlnLdKWoiASdBhGhzLyzLwPbNeGRd75h1rLdXkeqESp0EQlKUeGhvDyuL1d2juex99czdcEOryNVOxW6iAStyDAfU8b24foezfnvf2/mL/M2B/VDMnS3RREJamG+EJ4Z05MGEaG88OUOjuYW8IdRXfGFBN9DMlToIhL0fCHGn27sSqOoMF78agfHcvN5ekxPIkJ9XkfzKxW6iNQJZsYvhneicf1w/vjxJo6fXsnUsX2IjgzzOprfaAxdROqUuy5ty5O39mD5zmzG/H0ZmSfOeB3Jb1ToIlLn3NynJS+NS2HXkVPcNGUJO7JOeh3JL1ToIlInDe0Yz1uTBnCmoIibpyxhRXq5dwkPGCp0EamzureM5b17BtE4KpyxLy0P+EfaqdBFpE67qEl9Zt87iJ6tYnnwrTW88OX2gJ2rrkIXkTovNiqcWXf1Y1TP5vxl3hZ+/s+15BUWeR3rvGnaoogIJQ+ffmZMT9o0rc8zn29jb3YuU3/ah8b1w72OVmk6QxcRKWVmPHRlB567rRdrMo4x+oXFbDmY43WsSlOhi4h8zw09mvP2pAGcLijiphcX8+mGwHhYhgpdRKQMvVo34qP/HEz7+AZMmpXG3+Zvq/VflqrQRUTKkRgTydt3D+TGXi148rOtTH4tjZwzBV7HKpcKXUTkHCLDfDz1ox48dl0XPt+UyegXFtfaK0tV6CIiFTAzJg5uw2sT+3Mst4BRzy9m7voDXsf6ARW6iEglDWzXhI/uLxlXn/zaKv44ZyMFRcVex/pOhYVuZtPNLNPM1pez3MzsOTPbbmZrzay3/2OKiNQOzWPr8c7dAxk38CJe+jqd26Yt48Dx017HAip3hj4TGH6O5SOA5NKfScCUqscSEam9wkNDeHxUV567rRcbD5zg2mcX8eXmTK9jVVzozrmFwLluQzYKeNWVWAbEmlkzfwUUEamtbujRnI/uH0xCw0gmzFzJnz/Z5OkQjD/G0FsAe896nVH63g+Y2SQzSzWz1KysLD9sWkTEW+3iGvD+fZfwk/6t+fvCndwydSl7juR6ksUfhV7Wk1bLnH3vnJvmnEtxzqXExcX5YdMiIt6LDPPxpxu78cJPerMz6yTXPreI91fX/K14/VHoGUCrs163BPb74XNFRALKyO7N+PeDl9IpMZqH3l7DQ2+t5vjpmrsQyR+F/iFwR+lslwHAcedc7ZugKSJSA1o2iuKtSQN4+MoOfLT2ANc+u4hlO4/UyLYrM23xTWAp0NHMMsxsoplNNrPJpat8AuwEtgP/AO6ttrQiIgEg1BfCg1cm8+7kgYT5jNv+sYwnPt7ImYLqvce6eXWzmZSUFJeamurJtkVEakpufiFPfLyJ15fvoX18A576UQ+6t4y94M8zszTnXEpZy3SlqIhINYoKD+WJG7vxyp39OHmmkBtfXMLLX6dXy7ZU6CIiNWBIhzjmPXwZo3o0p03TqGrZhh5BJyJSQ2LqhfHUmJ7V9vk6QxcRCRIqdBGRIKFCFxEJEip0EZEgoUIXEQkSKnQRkSChQhcRCRIqdBGRIOHZvVzMLAvYfYH/eFPgsB/jBIq6uN91cZ+hbu53XdxnOP/9vsg5V+YDJTwr9Kows9Tybk4TzOriftfFfYa6ud91cZ/Bv/utIRcRkSChQhcRCRKBWujTvA7gkbq433Vxn6Fu7ndd3Gfw434H5Bi6iIj8UKCeoYuIyPeo0EVEgkTAFbqZDTezLWa23cx+6XWe6mBmrczsSzPbZGYbzOzB0vcbm9lnZrat9NdGXmetDmbmM7PVZjan9HUbM1teut9vm1m41xn9ycxizexdM9tceswH1oVjbWYPl/75Xm9mb5pZZDAeazObbmaZZrb+rPfKPL5W4rnSfltrZr3PZ1sBVehm5gNeAEYAXYDbzKyLt6mqRSHwM+dcZ2AAcF/pfv4SmO+cSwbml74ORg8Cm856/R1yKiAAAALfSURBVD/A06X7fRSY6Emq6vMsMNc51wnoQcm+B/WxNrMWwANAinOuK+ADfkxwHuuZwPDvvVfe8R0BJJf+TAKmnM+GAqrQgX7AdufcTudcPvAWMMrjTH7nnDvgnFtV+vscSv4Db0HJvr5SutorwGhvElYfM2sJjAReKn1twBXAu6WrBNV+m1lD4DLgZQDnXL5z7hh14FhT8gjMemYWCkQBBwjCY+2cWwhkf+/t8o7vKOBVV2IZEGtmzSq7rUAr9BbA3rNeZ5S+F7TMLAnoBSwHEpxzB6Ck9IF475JVm2eAXwDFpa+bAMecc4Wlr4PtmLcFsoAZpcNML5lZfYL8WDvn9gF/BfZQUuTHgTSC+1ifrbzjW6WOC7RCtzLeC9p5l2bWAHgPeMg5d8LrPNXNzK4DMp1zaWe/XcaqwXTMQ4HewBTnXC/gFEE2vFKW0jHjUUAboDlQn5Lhhu8LpmNdGVX68x5ohZ4BtDrrdUtgv0dZqpWZhVFS5q8752aXvn3o279+lf6a6VW+anIJcIOZ7aJkOO0KSs7YY0v/Wg7Bd8wzgAzn3PLS1+9SUvDBfqyvBNKdc1nOuQJgNjCI4D7WZyvv+Fap4wKt0FcCyaXfhIdT8iXKhx5n8rvSceOXgU3OuafOWvQhMK709+OAD2o6W3Vyzv3KOdfSOZdEybH9wjl3O/AlcEvpakG13865g8BeM+tY+tYwYCNBfqwpGWoZYGZRpX/ev93voD3W31Pe8f0QuKN0tssA4Pi3QzOV4pwLqB/gWmArsAP4P17nqaZ9HEzJX7PWAmtKf66lZDx5PrCt9NfGXmetxn8HQ4E5pb9vC6wAtgP/BCK8zufnfe0JpJYe7/eBRnXhWAOPA5uB9cAsICIYjzXwJiXfExRQcgY+sbzjS8mQywul/baOkllAld6WLv0XEQkSgTbkIiIi5VChi4gECRW6iEiQUKGLiAQJFbqISJBQoYuIBAkVuohIkPh/7UJsojncJLUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxU1f3/8dfJvkIICQGysISwyU4ERKGooIB8wVZUELVYlS5av3X9an+tdfm2ta5Vy9dKUVyqgqAVqgG0IIKAQECEEIGEsGQSIIGQFbLO5/fHDW0ICZmQmUxm8nk+HjzInXtm7uc+Lry5nDnnHiMiKKWU8nw+7i5AKaWUc2igK6WUl9BAV0opL6GBrpRSXkIDXSmlvISfuw4cFRUlPXv2dNfhlVLKI23fvv2EiEQ3tM9tgd6zZ09SU1PddXillPJIxpjDje3TLhellPISGuhKKeUlNNCVUspLaKArpZSX0EBXSikvoYGulFJeQgNdKaW8hAa6Ukq1lupK+Pw3UGRzycdroCulVGsoOwHvzIBNr0LG5y45hNtmiiqlVLtxLA0+mA1leXDDGzB4pksOo4GulPJ+1RXw7g/Btq1VDnfeOnA1VUhYDDJ3Jb5xI1x2XA10pZT32/QqHN4IyXdCUIcWfZQIlFVWU1JeTUl51bm/V1RzprKGimr7Oe+pEH8+OHEVeX85SoDfSp6cfgmzRyW0qI6GaKArpZzj4HroGA+RvRpvczwdstZd/DECw2DwTeAf5Ph7Th2G9c/DgOkw7UWH31ZjF3ILz7D/eAkZeaVkHC8lM6+EzLxSyiprzmkbFRZAbEQw3WOCiQoLJDI0gE4h/vj6GADCgLuqajhTaed0VTX9uoY7Xn8zaKArpVrGXgNrnoSNL0NUX/jZRvALOL/drqWw/B6oqWjZ8ba/DbPeg/CujrVf9SgYH5j8x/N2lVZUszHzBJl5pRSUVVJQVsnx4nJsp86QW3iGavt/Ok+6hAfSNyacG5PjSYwOJS4yhPhOwXSPCCYkoG1EaduoQinlmcqL4aO7IGM1JF4NB9bA5r/AuAf+08Zuh7VPw9cvQo/L4YevQ+BF3qFmrYNPfg4LroTZ70P34Rduv28V7EuBSU9RHdadw/nWnXbG8RI2Z51k26ECqmqs0A4J8CUyNIDo8ECGxUcwbUg3EiJDSIoJo090OB1D/C+u5lZkRM7rvm8VycnJos9DV8qDFWRZIzdOZMDUZ+HSu2DxHDiwFu7ZChHxUFECH90N+1fCyDtgyrMN3703x9FdsPgWKDkGIZENNhGBiuoafCqLyTUxzJRnOVku1I27fjHhTOgfzZX9ujA0LoLgAN+W1dVKjDHbRSS5wX0a6EqpZsv6Cpb+2Pr5xreh9w+snwuPwPzRkHgVXPO/tYG/HyY/A6PuBmOcc/zSfNj4Z6gs/fdLlTV2jhWVk3PqDNmnTlNdIwQFBZAWO4uqTklEhgYQHxlC35gwEqPDCA30zA4KDXSlVPOIQOYaa9x0fYVH4KtnoXMfuGUxRPY+d/+GF60+9YBw8PGFm96G3hNcUubx4nJWpR1jZdpRUg+dotouhAf5MWVQV24YEcelPSPx8XHSPyJtxIUC3aF/oowxk4GXAV9goYg8U29/AvA2EFHb5lERSWlR1Uop99n5nvUFZmOSroUbFjY8BPCyeyHtI6ipgtkfQOdEp5aWW3iGlWnHWLn7KKmHT1nldAnj7vG9ubJfF0YkRODn2z4nwTcZ6MYYX2A+MAmwAduMMStEJL1Os98AH4rIa8aYgUAK0NMF9SqlXO10AXzxOMSPgR+9fv5+4wsd4xrvPvELgLvWgK+/dYfeAiKC7dQZvrMVsstWxNaDBezMLgSgf9dwHpjUl6mDu9Kni2uGAXoaR+7QRwGZIpIFYIxZDMwA6ga6AGf/qe4I5DqzSKVUK1r7NJwphOtegE49L+4zmjNOHCirqCYjr5TTFdWcqarhWHE5Ww8W8E3WSY4XW8McA3x9GNi9Aw9f248pg7rSOzrs4mrzYo4EeiyQXWfbBoyu1+YJ4HNjzC+BUGBiQx9kjJkHzANISHD+LCmlVAvlbIfURTDm59B1kEsPdbToDJ/tOsq6fflsPVhAZc25syujwwMZ3SuS0b0iGRbfiX5dwwnwa59dKY5yJNAb+n9V/W9SZwNvicgLxpjLgHeNMYNE5JwrJCILgAVgfSl6MQUrpVzEXgOfPQhhMTDhMZcdJj23mIUbsljxXS7VdqFvTBh3XN6T5J6RhAf5ERLgS6eQAOI6BWOcNSqmnXAk0G1AfJ3tOM7vUrkTmAwgIpuNMUFAFNDAV+RKqTbn7Hjx3G+tpwG28Hkn9YkIGzJOsGB9Fl9nniAkwJfbL+vJj8f2oEfnUKceqz1zJNC3AUnGmF5ADjALuKVemyPA1cBbxpgBQBCQ78xClVIuUnDwP+PFpzznlEe7Fp2uIjO/FNup09hOneGf3+Wy91gJXcIDeWRyP+aM6uERMy89TZOBLiLVxph7gdVYQxLfFJE9xpingFQRWQE8CPzNGHM/VnfMXHHXAHel2isR2DwfTmZak3oCHfjS8NDXsOQ2EDvc9vFFjxcvr6phQ8YJNmae4Jusk+w7XnLOrMz+XcN5buYQpg/rTqCfZ8zI9EQ6sUgpb1B1BpbfC2nLrO2YQdYY8IgLDD5IfRNSHrYmBs1e3Ozx4na78OW+PJbvzGXN98cpq6whyN+HkT06MbpXZwbFdiCuUwixEcEeOyuzLWrxxCKlVBtWnGs92yR3J1z9OHQdCst+Yj3AatqL1pec9e1eCtsWQtI1tROEOjp8OLtdWLXnGK+syWDvsRI6hfjzX0O7M3VwN8b07qwjUdxIA10pT2bbboV5ZSnMeh/6T7Vev3sNvH8zfHh74+8d+0uY+GSzJv/sO1bC/Ut2kn60mN7Robx081CmDemOfzudmdnWaKAr5al2fWh1s4THwG2fQ8wl/9kXlQQ/XV+75FoD3aohnaHbUIcPJSIs2ZbN71bsITzIn5dnDWPakO7/XsBBtQ0a6Ep5Grsd1j4FX78EPa6Am96B0M7ntwsMg8QrL+oQldV2MvNKOVJwmpzCM3yTdZIv0o9zRZ8oXrp5GNHhgS08CeUKGuhKeZLyYvh4nnOfL47VL77tUAEr047xbXYh3x8tprLOuphhgX48dE1ffjGhj9c9vdCbaKAr5Snqjhef+rz1fPEWyi+p4L0th/loh43sgjME+/syJK4jP76sB4PjIujVOZS4TsFEhPjrrE0PoIGulCc4uN76glOkRePFz8orLuf19Vm8t+UwFdV2xiZ25v6JfZk8qGubWR9TNZ9eOaXaum0LYeX/QGRii58vXlVj57V1B5j/ZSbVduH6YbHcc2WiPrnQS2igK9WWff5b2PTKRY0Xr29PbhEPL91F+tFirhvSjYev6UfPKH2OijfRQFeqrSo7AZtehaGzYcb8i14sYretiHe/OcTHO3KICAng9dtGcu0lXZ1crGoLNNCVaqv2rwbEejZ5M8M8r6Sc1WnHWLbdxne2IoL9fZk1Kp6HrulHREjLR8WotkkDXam2av9K6BALXYc41LyiuoblO3NZtt3GtkMFiEDfmDCenH4JPxwRS4cgfbqht9NAV6otqiqHzLUwdFbja3fWKjpdxd+3HOatTYfIL6kgMTqU+65K4roh3egbo2tttica6Eq1RYc2QFUZ9JvSaJPqGjvvbz3Ci1/sp/B0FeOSonjxpqFc0SdKx4y3UxroSrVF+1aCfyj0HNfg7g0Z+Tz9aTr7j5cyNrEzv546gEGxFz8CRnkHDXSl2hoR2L8K+lwF/kHn7NptK+JPq/bydeYJ4iODef22kVwzMEbvyBXgYKAbYyYDL2OtWLRQRJ6pt/8l4OxTgEKALiIS4cxClWo3ju2C4hyqxj9G2pFT7DtWQkZeKem5xWzOOklkaACPTxvInDEJuvqPOkeTgW6M8QXmA5OwFozeZoxZISLpZ9uIyP112v8SGO6CWpXyamUV1Ww6cBKf9W9xJYbLP/Ynz74JgCB/H/p0CeO+q5O4e1wvwnXEimqAI3foo4BMEckCMMYsBmYA6Y20nw38zjnlKeX9TpZW8D8f7Wb9/nyqaqpJCVxLRuBAZo4ZxpC4jgzs1pHYTsH67HHVJEcCPRbIrrNtA0Y31NAY0wPoBaxtZP88YB5AQsIF1jpUqp04XlzOnIVbyC44zZ2jornzxLNEZR+CSS/wyKX93V2e8jCOBHpDtwWNrSw9C1gmIjUN7RSRBcACsBaJdqhCpbxUdsFp5izcwsnSCt6/MZaRm34Oeelw7R8g+U53l6c8kCOBbgPi62zHAbmNtJ0F3NPSopTyWCLW81c2PA81VY02qxEhqtrOaiAgwAffTyohIAzmLIU+E1uvXuVVHAn0bUCSMaYXkIMV2rfUb2SM6Qd0AjY7tUKlPEVVOfzzv2HXYiuUuww4r4kI7MopZOvBU4QG+nLtwK4EhwWAjx8MuxWi+rihcOUtmgx0Eak2xtwLrMYatvimiOwxxjwFpIrIitqms4HFIqJdKap9KLJZE4Ckdqm2XR9CTipM+DX84JHzpuwfLy7n0Y928eX+fKYM6sozNwyhY7COVlHO49A4dBFJAVLqvfZ4ve0nnFeWUm3coa+tFYROn/zPawFh1oLNA2ec07TGLryz+RAvfL6fqho7T8+4hFvH9NDJQMrpdKaoUs2VughSHoJOveD2FdChu/W6f8h5MzsPnSjj3g92kJZTzPi+0Tw94xJ6dNZFJZRraKAr1RzfvAarHrX6yG94A4IbnxCdllPE3EVbqbYLf7llONcN7qZ35cqlNNCVclRNFXz9Z+uBWbd8eMFFJzZmnmDeO6lEhASw5M5RJOqanaoVaKAr5aj9q6D0GEx7qcEwFxG+zS7ko+02lqba6BUVyts/GUXXjkENfJhSzqeBrpSjUhdBeHdrweY67Hbhox02XvvqAFn5ZQT5+zBtaDd+N+0SOoboKBbVejTQlXJEwUE4sAYmPAa+//lrs/1wAU/+M51dtiIGx3bkTzcMZurgbvrwLOUWGuhKOWLH22B8YPhtgPVkxP/9LJ0PtmYT0yGQl24eyoyhsfjoA7SUG2mgK9WU6krY8S70nQIdY9lx5BQPLNnJ4YLT/HR8b+67OonQQP2rpNxP/xQq1ZS9/4TTJ6gYdjsvr9rL6+uz6NohiMV3j2F0787urk6pf9NAV+pCDm+GlEc4ExbP5BW+HD51gJkj43j8vwbSQfvJVRujga5UY3a8g3z6APm+McwuvQ+/KH8+uHsElyXqXblqmzTQlWrIV8/Cl79nu89Q7iq9h7lXD+cXE/oQ4Ofj7sqUapQGulL1Hd2F/cs/8qn9cv7k/ysWzEtmVK9Id1elVJM00JWqo6q6mtx3f06ohLEq4UH+OXsckaEB7i5LKYfo/x+VqlVQVskbrz5Nj9NpfJP4K179yVUa5sqj6B26UlhPRnzk3S9578wbnIgaybTbHjhvgQql2jqHAt0YMxl4GWvFooUi8kwDbW4CnsBaQPo7ETlvmTql2pwNL5KTtoGjR0v4s+8JInxOY256VcNceaQmA90Y4wvMByZhLRi9zRizQkTS67RJAh4DLheRU8aYLq4qWClnKTuWSeiaJxGJom9AOLGdgjGX/gliLnF3aUpdFEfu0EcBmSKSBWCMWQzMANLrtLkbmC8ipwBEJM/ZhSrlTBsy8jm0+PfcIoYVI95k3rRx+PnqV0rKszkS6LFAdp1tGzC6Xpu+AMaYjVjdMk+IyCqnVKiUE9XYhd+tSGPJN1lsCV5DSY+J/GLGD9xdllJO4UigN9SZKA18ThIwAYgDNhhjBolI4TkfZMw8YB5AQkJCs4tVqiVEhF9/vJslqdm8MPAwkVlFMO6n7i5LKadxJNBtQHyd7Tggt4E234hIFXDQGLMPK+C31W0kIguABQDJycn1/1FQqmUqy2D9c1Cab237+sGoeRBzCSLCH1fuZUlqNvde2Ycbjr4GEQmQeJV7a1bKiRwJ9G1AkjGmF5ADzALqj2D5BJgNvGWMicLqgslyZqFKNemrP8HGl6FDnLVdXgi7llJ9/ev85Wg/FqzP4vbLevDgCAPzN8DVj19wXVClPE2TgS4i1caYe4HVWP3jb4rIHmPMU0CqiKyo3XeNMSYdqAEeFpGTrixcqXPk7YXN82H4rTBjvvVSziFqPriFbktvpbrqRuYMmsMTV8VgNjwHPn4w7FY3F62UcxkR9/R8JCcnS2pqqluOrbyMCLw1DY6nUf6zrXx+uIaPttvYkJFPAJW8GfkuY8vWnPuegTPgpnfcU69SLWCM2S4iyQ3t05miyuPJrg8xh7/mH7EP8fifd1JSXk33jkH8fEIiNycnkBB5Pez9FEqO/edN/ae5r2ClXEQDXXm0XQeOEPfJIxyx9+Y3R0Zy7aAYbhgZx2W9O5+7vueA/3JfkUq1Eg105bHW7csj+72HGeRTxI7xf2XLuEmE6dqeqh3TP/3KI/3jWxuLli3nH/6fUzF0LhMnTnF3SUq5nQa68hh2u/BVRj5/33yYtXuP8Xn4uxj/SIIn/87dpSnVJmigqzavusbOh6k2/vrVAY4UnCYqLJDXB+0lKfN7uO6vENzJ3SUq1SZooKs2S0RYlXaM51bvI+tEGcMTInhkcj+uSfAh4PW7IWEsDJ3l7jKVajM00FWbdLy4nIeWfseGjBMkdQnjb7cnM3FAF0xeOrw1C6pOw3XP63PLlapDA121OSm7j/Lrf+ymosrO0zMu4ZbRPfD1MbD3M/h4HgSEwdwUfW65UvVooKs2w3bqNH9I+Z6U3ccYGteRl24eRu/oMGvnnk9g6VzoPgxmvQ8duru1VqXaIg105XanK6v567oDvL4+C2PgwUl9+dmERPzPLjhRXgQrH7HC/I6V4B/s3oKVaqM00JXbiAjLd+byzMq9HCsuZ/rQ7jw6pT/dI+oF9pd/gNI8mL1Yw1ypC9BAV62uorqGzQdO8sqaDHYcKWRwbEdevWU4l/aMPL/x0V2wdQEk/wRiR7R+sUp5EA101WpWpR1l2XYbGzNPcqaqhqiwQJ6dOYSZI+LOfe7KWXY7fPYABEfC1b9t/YKV8jAa6Mrlik5X8dvlaaz4LpfYiGBuTI7jyn5duCyxM0H+jSwwIQJfPQO2bXD9azp5SCkHaKArl9qQkc8jy3aRX1LBg5P68vMJifid/bKzMVVnYMUvYfdSGHwTDJ3dOsUq5eE00JVLpOcW8+zqvazbl0/vqFA++vlYhsZHNP3G4qOw+BbI3QFX/QbGPaSTh5RykEOBboyZDLyMtQTdQhF5pt7+ucBzWGuOAvxFRBY6sU7Vxh0+WcbunCIyjpeSllPE2n15dAjy57Ep/fnx2J6Nd63UlbMdFs+B8mJrrHn/61xfuFJepMlAN8b4AvOBSYAN2GaMWSEi6fWaLhGRe11Qo2qjKqvtrN5zjHe/OczWgwUA+BhIiAzhZz9I5Gc/SKRjsL9jH7ZrKSy/B8Jj4K4vdBaoUhfBkTv0UUCmiGQBGGMWAzOA+oGu2pFNmSd44MPvOFZcTnxkMI9O6c/4pGh6R4c6djd+lt0Oa5+Gr1+0HrZ187sQGuW6wpXyYo4EeiyQXWfbBoxuoN0NxpjxwH7gfhHJrt/AGDMPmAeQkJDQ/GqV21XX2HllTQavfplJYnQYi350KT/oG93wsMOmVJTAR3fD/pUw4naY+gL4BTi/aKXaCUcCvaG/qVJv+5/AByJSYYz5GfA2cNV5bxJZACwASE5Orv8Zqo0rOl3FvHdT2XKwgBtHxvHkjEsICbjI79VPHYIPZkP+PpjyLIyap19+KtVCjvxttAHxdbbjgNy6DUTkZJ3NvwF/anlpqi0pKa/i9kVb+T63mBduHMoNI+Na9oEfz4PiHLj1I0i80jlFKtXONTEgGIBtQJIxppcxJgCYBayo28AY063O5nTge+eVqNytrKKaOxZtY09OEfPnjGh5mB9Lg+wt8INHNcyVcqIm79BFpNoYcy+wGmvY4psisscY8xSQKiIrgPuMMdOBaqAAmOvCmlUryik8w4Mf7uTb7EJenT2cSQNjWv6h2xeBb6CuNqSUkznUASoiKUBKvdcer/PzY8Bjzi1NuVNaThF/25DFp7uOYoAXbxrK1MHdmnxfkypK4bslMOhHENLAw7iUUhdNZ4qqc+w4coqX/5XBV/vzCQ3w5Y6xPbnjil7E1n+k7cVKWwaVJTDyDud8nlLq3zTQFSLCN1kF/N+6TDZknKBTiD8PX9uPW8f0cHxikKNSF0GXgRA/yrmfq5TSQG/PauzCZ7uP8rf1WezOKSIqLIDHpvTn1jE9CA10wR+NnB1wdCdM1cWdlXIFDfR26lhROfct/patBwvoHR3KH380mB8Oj23eLM/msNth0yvgHwJDbnLNMZRq5zTQ26G1e4/z4IffUVFt57mZQ7ihsQUmnKWiFP7xU9j7KYx7EII6uu5YSrVjGujtyMETZby6NoOPd+QwoFsH/nLLcBKjw1x70FOHrcfh5qXDtX+AMb9w7fGUasc00L1ccXkVu21FLNtuY/nOHAL8fPjp+N7cP6mv67pXzjq8GZbcCjVVMGcp9Jno2uMp1c5poHuhqho7r607wCff5pB1ogyAYH9f7hrXm7vH9SY6PND1Rex4Bz59ADr1gNmLISrJ9cdUqp3TQPcyWfml3L9kJ9/ZihiXFMWPRsQyJC6CofERzh2CWFEKG/8MpXnn7ys7Afs+g8SrYOabuh6oUq1EA91L1NiF97Yc5o8pewnw8+H/5oxwzszOhhQesZ6UmJcOoV3O328MjP0lXP0E+OofMaVai/5t8wJpOUX8v0/S+C67kHFJUTw3cyhdOwa55mDaL65Um6WB7sGOnDzNgg0HeH/LESJDA3h51jCmD+2OcdWknWO74Z3pEJGg/eJKtUEa6B4oLaeI19YdYGXaUXx9DLeMTuDha/rTMcTJ0/TrstutLzkDO8BPPofQzq47llLqomige5Ci01U8u3ov7289QligH/PGJzJ3bE/Xda/UtfM9sG2FGf+nYa5UG6WB7gFEhOU7c/nfz9IpKKvkjrG9uH9SEuFBLrwjr+t0AXzxOCRcBkNnt84xlVLNpoHexmXll/Lb5WlszDzJ0LiOvHXHKAbFtvLU+TVPQnkRXPcC+DiyyJVSyh0cCnRjzGTgZawVixaKyDONtJsJLAUuFZFUp1XZzogI+46XsHxnLm9sOEigvw9PXz+IW0Yl4OvKZ640JG8vbH/bmrIfc0nrHlsp1SxNBroxxheYD0zCWjB6mzFmhYik12sXDtwHbHFFoe1BQVklf/7Xfr5IP87RonIApg/tzm+mDaBLeCv0kzfk+9rlY6/4lXuOr5RymCN36KOATBHJAjDGLAZmAOn12j0NPAs85NQK24kv9+XxyLJdFJ6u5Or+MfxqYjQ/6Nuldb7wvJB9KyEuGcIamECklGpTHAn0WCC7zrYNGF23gTFmOBAvIp8aYxoNdGPMPGAeQEJCQvOr9RLpucV8sjMHAwT5+5JTeIZl2230jQnj7TtGMbB7B3eXaCk+Crk74OrHm26rlHI7RwK9oU5b+fdOY3yAl4C5TX2QiCwAFgAkJydLE829TlpOES+vyeCL9OME+PpgDFRU2/ExcOcVvXj42n6ufwJic+xfZf3ed4p761BKOcSRQLcB8XW244DcOtvhwCBgXe0Mxa7ACmPMdP1i1HK6sprff/Y97205QocgP+6f2Je5l/ekY7A/drtQbRcC/Nrg6JF9KyGiB3QZ4O5KlFIOcCTQtwFJxpheQA4wC7jl7E4RKQKizm4bY9YBD2mYW77LLuRXS3Zy6GQZd13Ri/smJtGhzvhxHx9DQGuPXHFEZRkc/ApG3qHrfyrlIZoMdBGpNsbcC6zGGrb4pojsMcY8BaSKyApXF+kp7HZhV04R2w4WkJFXQkZeKbtsRcSEB/LeXaMZmxjV9Ie0FVnroLoc+k12dyVKKQc5NA5dRFKAlHqvNfhNmYhMaHlZniU9t5iFG7L4an8+J8sqAYgKCySpSxh3jevFLyb0ce6zyFvDvhQI7Ag9Lnd3JUopB+lM0RbaknWSO99OxcfAVf27cGX/LlzeJ4qosFZYFchV7HbYvxqSJoKvh/1DpFQ7poHeAl/tz+en76YSGxHMe3eNcf+YcWdJWwZl+Tq6RSkP0waHVniGT3flctfb2+gdFcaSn17mHWEuAl89Bx/fDbEjof9Ud1eklGoGvUNvpuoaO8+t3sfr67MY2aMTb869tG32jxflgG1b896Tvhz2fAyDb4Lpr4B/sGtqU0q5hAZ6M5woreCX73/L5qyT3Domgd9OG0igXxuaCHTWgbWwdK71hMRmMTDxCbj8VzpUUSkPpIHuoKoaO3MXbSXjeCnP3ziUmSPj3F3S+URgy+uw+tcQ3Q/mLIOAMMffHxwBHbq7rj6llEtpoDtowfos0nKKeW3OCKYM7ua+QsqLYOX/QP7e8/dVV0BeOvS7Dn70OgSGt359Sim30UB3QMbxEl7+VwbXDe7m3jA/eQDevxlOHYTeE8A08J324ButLhNdiEKpdkcDvQk1duHhZbsIDfTlyRmNLPBQdcYa5hfhwidIHvgSlv4YfPzg9hXQUyf8KKXOpbdxTVi08SA7swt5YvolDU8WErG+gHxlOKQuck0RBVnWnXmHWLh7rYa5UqpBGugXsMtWyLOr9jFpYAzThzbyZeG+FOsxsx26w6e/gpSHoabaeUWIQMoj4BsAt34MnXo677OVUl5Fu1waUXS6il+8t4OosACevWEIpqFhfJWnYeWjED0A5q2DtU/D5r/Aoa8hvOv57YM6wtW/g8hejhey91PI/AKu/SN0cGP/vVKqzdNAb4CI8NCy7zhWVM6HP7uMTqEBDTfc8DwUHYG5KeAfBNf+HmIGwfa3oKLk/Pa2VMj6Cm5+F3pe0XQhlWXWPxgxg2DUvBadk1LK+2mgN+CNrw/yRfpxfjttICMSOjXcKH8/bHwFhs4+t0972GzrV0NOHoAPZsE7M+DaP0DCmAsX8u3fodgGM98AX71USqkL05So52RpBS98vp+JA2L4yeU9G2+44QVravykpxz/8M6JcNe/YNmdsPIRx94zbE7TweRZmLUAAA5tSURBVK+UUmign+etTYcor67h0Sn9Gu43B+uLyoPrIWkShHVp3gGCOsItS6zVgKrOXLitb4A13lwppRzgUKAbYyYDL2OtWLRQRJ6pt/9nwD1ADVAKzBORdCfX6nIl5VW8tekQky/pSp8uF5hleeoQlORCj7EXdyAfX0i86uLeq5RSjWhy2KIxxheYD0wBBgKzjTED6zV7X0QGi8gw4FngRadX2gr+/s0RSsqr+cWEPhdueHiT9buu5qOUakMcGYc+CsgUkSwRqQQWAzPqNhCR4jqboYA4r8TWUV5VwxtfZzEuKYrBcR0v3PjwRgiOhKh+rVOcUko5wJEul1ggu862DRhdv5Ex5h7gASAA8Lj+hKWp2ZworeSeK5u4Owcr0HuM1eelKKXaFEcSqaFvBs+7AxeR+SKSCPwP8JsGP8iYecaYVGNMan5+fvMqdaHqGjt//cpasGJ0r8gLNy7KsfrQtbtFKdXGOBLoNiC+znYckHuB9ouB6xvaISILRCRZRJKjo6Mdr9LFVqYdI6fwDD8d37vxkS1nHdls/X6xX4gqpZSLOBLo24AkY0wvY0wAMAtYUbeBMSapzuZ1QIbzSnQtEeFvG7LoFRXKxAExTb/h8EYICIeug11fnFJKNUOTfegiUm2MuRdYjTVs8U0R2WOMeQpIFZEVwL3GmIlAFXAK+LEri3amrQcL2GUr4unrB+Hj48Cya4c3WRN9fNrg0nNKqXbNoXHoIpICpNR77fE6P/+3k+tqNX/bcJBOIf7MHOHAknJlJ6yVgobc7PrClFKqmdr1MI2s/FLW7D3OrWN6EBzgwB23jj9XSrVh7Xrq/xtfH8Tfx4fbLuvReKN9K8G2zfr58CbwC4Luw1unQKWUaoZ2G+gFZZV8tMPG9cO70yU86PwG9hr41xOw6RVr7c6z63cOugH8GnmcrlJKuVG7DfR3Nh+ivMrOvPG9z99ZXgQf3QUZn8Old8PkP4Kvf6vXqJRSzdEuA/1MZQ3vbD7M1f27nP8QrpMH4IPZUHAArnsBLr3LPUUqpVQztctAX7Y9m4KyyvPvzrO+gg9vB2Pgtn9Ar/HuKVAppS5CuxvlUmMXFn59kKHxEYyqO80/9U1494cQ3g3u/lLDXCnlcdpdoK/ec4zDJ0+fO80/91v49AHrGeV3ft68RZyVUqqNaFeBLiK8vj6LHp1DuPaSrtaL9horzEOjrbU7gzq4t0illLpI7SrQd9mK+C67kDuv6IXv2Wn+O96G3B1w7e+t5eGUUspDtatAX7o9m0A/H64fHmu9UHYC/vUk9BwHg290b3FKKdVC7SbQy6tqWLEzl8mDutIhqHZM+b9+B5WlMPV5a2SLUkp5sHYT6J+nH6e4vJobR9Y+2v3IN/Dt3+Gye6BLf/cWp5RSTtBuAn1pajaxEcGMTewMNdXw2YPQIRbGP+Lu0pRSyinaRaDnFp7h68wT3DAi1nrm+ba/wfE0a0p/YJi7y1NKKadoF4H+8Q4bIjBzZDwUH4W1v4c+E2HAdHeXppRSTuP1U/9FhGXbbYzuFUlC5xBY9kuoqYQpz+oXoUopr+LQHboxZrIxZp8xJtMY82gD+x8wxqQbY3YZY9YYYy7wgPHWtfVgAYdOnubG5HgoyIK0ZTD2l9A50d2lKaWUUzUZ6MYYX2A+MAUYCMw2xgys1+xbIFlEhgDLgGedXejF+jDVRligH1MHd7UWqwAYcZt7i1JKKRdw5A59FJApIlkiUgksBmbUbSAiX4rI6drNbwAHFuh0vZLyKlJ2H+W/hnYjJMDPCvQuA6FTT3eXppRSTudIoMcC2XW2bbWvNeZOYGVDO4wx84wxqcaY1Pz8fMervEif7jrKmaoabkqOhzOnrCXk+k1x+XGVUsodHAn0hr45lAYbGnMrkAw819B+EVkgIskikhwdHe14lRdpybZskrqEMSw+AjL+BVID/aa6/LhKKeUOjgS6DYivsx0H5NZvZIyZCPw/YLqIVDinvIu3/3gJO7MLufnSeOsxuftSILQLdB/h7tKUUsolHAn0bUCSMaaXMSYAmAWsqNvAGDMceB0rzPOcX2bzfbgtGz8fww+Hx0J1JWSugb7Xgk+7GHqvlGqHmkw3EakG7gVWA98DH4rIHmPMU8aYszNzngPCgKXGmJ3GmBWNfFyrqKy2849vc5g4IIbOYYFwZBNUFGl3i1LKqzk0sUhEUoCUeq89XufniU6uq0XW7j3OybJKbrq0drDNvpXgFwS9J7izLKWUcimv7H9YmmqjS3gg45OiQcQK9N4TICDE3aUppZTLeF2g55WUs25/Pj8aEYefrw8c2gCFh3W4olLK63ldoP9jRw41duHG5DioqYKUhyGiBwy52d2lKaWUS3nVw7lEhKXbbYxIiCAxOgw2vgz5e2H2EvAPdnd5SinlUl51h74zu5DMvFLrQVxFNlj3J2tkS7/J7i5NKaVczqvu0JdutxHk78O0Id1g+U9A7DD5GXeXpZRSrcJr7tDPVNbwz525TB3UjfC87fD9Chj/IHRqM0/yVUopl/KaQE/ZfZSSimpmJsfBtoUQ2BHG3OPuspRSqtV4RaCLCIs2HaRPlzAu6wqkL4ehN+u4c6VUu+IVgZ56+BRpOcXMHdsTs/N9a4m5kXe4uyyllGpVXhHoizYepEOQHz8a3g22vwXxYyCm/qJKSinl3Tw+0HMKz7B6z3Fmj0ogJGcTFByA5J+4uyyllGp1Hh/o72w+hIhw22U9IHURBHeCgTOafJ9SSnkbjw7005XVLN6azbWXdCXO5xTs/RSGzQH/IHeXppRSrc5jA11EeH71forOVPGLAeXw5mTw8dPuFqVUu+Wxgf5/6w7w5saD/HHAIQavuhHsVXBHCnROdHdpSinlFg4FujFmsjFmnzEm0xjzaAP7xxtjdhhjqo0xM51f5rne/eYwz63ex6NJucw++Gvo0h/u/hJiR7r60Eop1WY1GejGGF9gPjAFGAjMNsbUHxN4BJgLvO/sAutbvjOHx5enMXFADPMitkJwJMz9DDp0c/WhlVKqTXPkDn0UkCkiWSJSCSwGzhlGIiKHRGQXYHdBjeeI6RDExAEx/GXWYHwyPrcWftZH4yqllENPW4wFsuts24DRF3MwY8w8YB5AQkLCxXwEY3p3ZkzvznDoaygv1JWIlFKqliN36KaB1+RiDiYiC0QkWUSSo6OjL+Yj/mPfSvANgMSrWvY5SinlJRwJdBsQX2c7Dsh1TTnNsG8l9BwHgeHurkQppdoERwJ9G5BkjOlljAkAZgErXFtWE05kWFP8tbtFKaX+rclAF5Fq4F5gNfA98KGI7DHGPGWMmQ5gjLnUGGMDbgReN8bscWXR7Euxfu+rS8sppdRZDi1BJyIpQEq91x6v8/M2rK6Y1rFvFXQdDBHxTbdVSql2wvNmipadhOxvrMWflVJK/ZvnBXrG59biz9rdopRS5/C8QA/qCP2ug27D3F2JUkq1KQ71obcp/adav5RSSp3D8+7QlVJKNUgDXSmlvIQGulJKeQkNdKWU8hIa6Eop5SU00JVSyktooCullJfQQFdKKS9hRC5qrYqWH9iYfODwRb49CjjhxHI8RXs87/Z4ztA+z7s9njM0/7x7iEiDKwS5LdBbwhiTKiLJ7q6jtbXH826P5wzt87zb4zmDc89bu1yUUspLaKArpZSX8NRAX+DuAtykPZ53ezxnaJ/n3R7PGZx43h7Zh66UUup8nnqHrpRSqh4NdKWU8hIeF+jGmMnGmH3GmExjzKPurscVjDHxxpgvjTHfG2P2GGP+u/b1SGPMF8aYjNrfO7m7VmczxvgaY741xnxau93LGLOl9pyXGGMC3F2jsxljIowxy4wxe2uv+WXt5FrfX/vnO80Y84ExJsjbrrcx5k1jTJ4xJq3Oaw1eW2N5pTbbdhljRjT3eB4V6MYYX2A+MAUYCMw2xgx0b1UuUQ08KCIDgDHAPbXn+SiwRkSSgDW1297mv4Hv62z/CXip9pxPAXe6pSrXehlYJSL9gaFY5+/V19oYEwvcBySLyCDAF5iF913vt4D6CyA3dm2nAEm1v+YBrzX3YB4V6MAoIFNEskSkElgMzHBzTU4nIkdFZEftzyVYf8Fjsc717dpmbwPXu6dC1zDGxAHXAQtrtw1wFbCstok3nnMHYDzwBoCIVIpIIV5+rWv5AcHGGD8gBDiKl11vEVkPFNR7ubFrOwN4RyzfABHGmG7NOZ6nBXoskF1n21b7mtcyxvQEhgNbgBgROQpW6ANd3FeZS/wZeASw1253BgpFpLp22xuvd28gH1hU29W00BgTipdfaxHJAZ4HjmAFeRGwHe+/3tD4tW1xvnlaoJsGXvPacZfGmDDgI+BXIlLs7npcyRgzDcgTke11X26gqbddbz9gBPCaiAwHyvCy7pWG1PYbzwB6Ad2BUKwuh/q87XpfSIv/vHtaoNuA+DrbcUCum2pxKWOMP1aYvyciH9e+fPzsf8Fqf89zV30ucDkw3RhzCKsr7SqsO/aI2v+Sg3debxtgE5EttdvLsALem681wETgoIjki0gV8DEwFu+/3tD4tW1xvnlaoG8Dkmq/CQ/A+hJlhZtrcrravuM3gO9F5MU6u1YAP679+cfA8tauzVVE5DERiRORnljXda2IzAG+BGbWNvOqcwYQkWNAtjGmX+1LVwPpePG1rnUEGGOMCan98372vL36etdq7NquAG6vHe0yBig62zXjMBHxqF/AVGA/cAD4f+6ux0XneAXWf7V2ATtrf03F6lNeA2TU/h7p7lpddP4TgE9rf+4NbAUygaVAoLvrc8H5DgNSa6/3J0Cn9nCtgSeBvUAa8C4Q6G3XG/gA6zuCKqw78Dsbu7ZYXS7za7NtN9YIoGYdT6f+K6WUl/C0LhellFKN0EBXSikvoYGulFJeQgNdKaW8hAa6Ukp5CQ10pZTyEhroSinlJf4/UEjHTBaIgwgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores2)\n",
    "plt.plot(test_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parameter ReLU + Sigmoid + Cross Entropy Cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net3(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net3, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(28*28, 30)\n",
    "        self.fc2 = nn.Linear(30, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.prelu(self.fc1(x), t.Tensor(1)) # weight参数不知道怎么用，随便写的一个\n",
    "        x = F.logsigmoid(self.fc2(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net3 = Net3()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net3.parameters(), lr = 1e-1)\n",
    "loss_scores3, train_scores3, test_scores3 = fit(net3, criterion, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxV9Z3/8dcnO2QhCQkJJIGEPUH2IJvraFW0al27WEWrpYxOrVVnqtZuM7/aUdtpaxWXce+CdcG1VuuCooJgEMIW1rCFQBIChLCGJN/fH/fKICUQyE1O7rnv5+ORB9x7j/e8jwffHr7ne84x5xwiIhL+orwOICIioaFCFxHxCRW6iIhPqNBFRHxChS4i4hMxXq04IyPD5efne7V6EZGwNH/+/K3OucwjfeZZoefn51NSUuLV6kVEwpKZrW/pMw25iIj4hApdRMQnVOgiIj6hQhcR8QkVuoiIT6jQRUR8QoUuIuITns1DP1Franbx6oJNFPVKoahnN/LSu2BmXscSEfFc2BX6ssqdPDhzNc3B27gnx8cwuGcyg7NTgr8mMzArmeSEWG+Dioh0sLAr9AuH9+LswixWVtWzbPNOllbWsWJLPa8s2ET9p40Hl8tJ7cLg7GQGZCUzKDuJgVnJ9MtMIiE22sP0IiLtJ+wKHaBLXDTD81IZnpd68D3nHBXb97JiSz0rqupZvqWeVVX1zFpVw4GmwOF8lEGf7okM6BEo+AFZgV8LMhJV9CIS9sKy0I/EzMhL70peelfOLso6+H5DYzPranezsqqelVvqWVm1i5XV9by3vJqm5v8r+vzuifTvkcSArCQG9AiUvY7oRSSc+KbQWxIXE8XArMC4OsP+7/39jU2U1+xmVfUuVlcFin7VEYq+d3pXBmQlf+moXkUvIp2R7wu9JfEx0RT2TKGwZ8qX3m9obGbt1t2sqq5nVbDkV1btYubyahoPO6IfkJXEoKwvxukDQzex0ZoJKiLeiNhCb0lcTBSDsgMFfagvin5lVWBsfmXVLlZW1fPOsqqDM25io42+GUkMzE5mUFYSg7JTGJSVTG5aF6KiNLVSRNqXCr2VWir6fQeaWFMTKPeVVbtYuaWez9dv5/XSyoPLJMZFMzA7MLWy8JAplimaWikiIaRCb6OE2GiG9OrGkF7dvvR+/b4DrKreFZh1s6Wess07eXPxZqbP23Bwmdy0LgeHfYp6pjCkVwq5abpQSkROjAq9nSQnxDKqdxqjeqcdfM85R9XO/ZRt3smyzTspC/68V/Z/wzYpCTEU9UoJ/k8ihZNyutE3I5EYjc2LyDGo0DuQmZHdLYHsbgmcObjHwff3NjSxfMvO4IVSO1lWuZM/fbqe/Y3NACTERlHYM4WTenVjaE43TsrpxoCsJJ2AFZEvMeecJysuLi52eqZoyxqbminfupullXUs2bSTxZvqWFa5k137A1fDxsdEUdQrhWE53Riam8rw3G70y0zSyVcRnzOz+c654iN+pkIPH83NjrW1u1myqY7FFXUsqqhjSWUdexqaAEiKj+GknBSG56UyIjdwJW3PbgkakxfxkaMVuoZcwkhUlNEvM3Bh08UjcgBoanaU1+yitKKORRU7KK2o46mP19HQFBiuyUqJZ0ReKiN7pzEyL5Vhual0idNFUSJ+pEIPc9FRFriSNSuZy0fnAoGrYMs217Nww3YWbtzBgo07eHtp1cHli3qmMKp3KqP6pDG6Txo5qZpZI+IHxxxyMbM84FkgG2gGHnPO/f6wZa4CfhR8uQv4V+dc6dG+V0MuHat2134WbtzB5xu2M3/9dko31rH3QGCoJjslgeL8NIr7pFGcn05hzxSiNRYv0im1dcilEbjNOfe5mSUD883sHefcskOWWQuc7pzbbmaTgMeAsW1OLiHTPSmeswqzOKswcOOyxqZmlm+pp2TdNkrWB0r+jUWbgcBY/Kg+aYwtSOfkgnSG5XYjPkbDNCKd3XGfFDWzV4EHnXPvtPB5GrDEOZdztO/REXrnU7F9DyXrtvPZum3MW7uNVdW7gMBVsiPyUhlXkM64vt0Z1SdNNycT8UjIZrmYWT4wCzjJObezhWVuBwY75244wmdTgCkAvXv3Hr1+/fpWr1s63rbdDQfLfe7aWpZV7qTZQVx0sOD7pjOuX3dG9VbBi3SUkBS6mSUBHwK/dM7NaGGZM4FpwCnOudqjfZ+O0MPPzn0HKFm3jbnl2/i0vJbFm+oCBR8TxejeaUzo150J/bszLDdVFz2JtJM2F7qZxQJvAG875/6nhWWGAS8Dk5xzK4/1nSr08Ldz3wE+W7uNOWtqmVNey7LNO3EucDOysX27M7F/Bqf0z2BgVpJm0YiESJtOilrgv8QngLKjlHlvYAZwdWvKXPwhJSH2Sydat+9u4NPyWj5Zs5XZq2t5f3k1AJnJ8ZzSP4OJ/TM4dUAGWSkJXsYW8a3WTFs8BfgIWExg2iLAXUBvAOfcI2b2OHAZ8MWgeGNL/wf5go7Q/W/Tjr18smorH6/eyiert1K7uwGAQVnJnDogg9MGZnJyQbrG30WOgy79F881NzvKtuzk41Vb+WjVVuat3UZDUzMJsVGMLejO6QMzOXNwDwoyEr2OKtKpqdCl09nb0MSna2v5cEUNs1bWUL51NwD53btyxqAenDm4B2N19C7yT1To0ultqN3DByurmbm8mtlratnf2EzXuGgm9s/g7MJAwfdI1ti7iApdwsrehibmlG/l/eXVvF9WTWXdPgCG56VyTlEW5xRl0b+HZs5IZFKhS9hyzlG2uZ73yqp4t6yK0oo6IDA0c+6QbM4Zks3IvFTdB14ihgpdfGNL3T7eLaviH8uqmLNmKweaHD2S4zl3SDaTTsrm5IJ0Pa5PfE2FLr5Ut/cAM5dX89aSLXywspp9B5pJT4zj3CHZfHVYT8aq3MWHVOjie3saGvlwRQ1vLtnCe2VV7GloontiHJOGZnPR8ByK+6RpWEZ8QYUuEWXfgSY+WFHN64s2815ZFfsONNOzWwIXDu/F10bkUNgzWSdUJWyp0CVi7d7fyLtlVby2sJIPV9bQ2OwYmJXEJSNz+drIXvTs1sXriCLHRYUuQuB2wH9bvJlXFmxi/vrtmMGEft25bFQu552UTdc4PZFROj8Vushh1tfuZsbnm3h5wSY2bNtDUnwMFwztyRXFuYzuk6YhGem0VOgiLXDOMW/tNl6YX8Gbizezp6GJ/j2S+MaYPC4ZmUP3pHivI4p8iQpdpBV272/kjUWVPPfZRhZs2EFcdBTnnZTNVWN7c3JBuo7apVNQoYscpxVb6pk+bwMvfV5B/b5GBvRI4urxfbh0VC5J8RprF++o0EVO0N6GJl5fVMmfPl3Pooo6EuOiuWx0LpMn5NMvM8nreBKBVOgiIbBw4w6enbOON0o309DUzBmDMrluYgGnDcjQcIx0GBW6SAjV1O/nL3M38Ke566mp38+grGRuOLWAi0b0Ij5G92+X9qVCF2kHDY3NvFZayeMflbN8Sz09kuO5/pQCvjW2N8kJsV7HE59SoYu0I+ccH63ayqOz1vDJ6lqSE2K4Znwfrj+lL+mJcV7HE59RoYt0kNKNO3jkwzW8tXQLCTHRXD2+DzecWqCnLUnIqNBFOtjq6noemrmGVxduIjY6im+P68O/ntGPDF2oJG2kQhfxyNqtu/nD+6t4ZcEm4mOimTwhn6mn9yW1q4Zi5MSo0EU8Vl6zi9+/t4rXSitJio9h6un9uG5ivm4IJsdNhS7SSazYUs/9by/n3bJqMpPj+eHZA7myOFdPVpJWO1qh60+RSAcalJ3M45PH8MLU8fRO78pdLy/mggc+5sOVNV5HEx84ZqGbWZ6ZzTSzMjNbamY/OMIyZmYPmNlqM1tkZqPaJ66IP4zJT+fFqeOZdtUo9h5oYvKT87juqXmU1+zyOpqEsdYcoTcCtznnCoFxwE1mVnTYMpOAAcGfKcDDIU0p4kNmxvlDe/LOrafx4/ML+Wzdds793Sx+9fcydu1v9DqehKFjFrpzbrNz7vPg7+uBMiDnsMUuBp51AZ8CqWbWM+RpRXwoPiaa757Wl/dvP52LR+Tw6IflnP2bD3lryWa8Oscl4em4xtDNLB8YCcw97KMcYOMhryv459LHzKaYWYmZldTUaMxQ5FA9khP49RXDmXHjBNIS45j6p8+5/pkSNm7b43U0CROtLnQzSwJeAm5xzu08/OMj/CP/dGjhnHvMOVfsnCvOzMw8vqQiEWJU7zRe/7eJ3H1BIZ+W13Lu72bx9CdraW7W0bocXasK3cxiCZT5n51zM46wSAWQd8jrXKCy7fFEIlNMdBQ3nNqXd249nTH56fz89WVc+egcVlfrpKm0rDWzXAx4Aihzzv1PC4u9BlwTnO0yDqhzzm0OYU6RiJST2oWnrxvDb64YzqrqXVzwwEc8+bGO1uXIWnOZ2kTgamCxmS0MvncX0BvAOfcI8CZwPrAa2ANcF/qoIpHJzLhsdC6nDszgzpcW859vLOOdZVXcf8UwctO6eh1POhFdKSoSRpxzvFBSwS9eX0qUGb+8dCgXDe/ldSzpQLpSVMQnzIwrx+Tx1i2nMTA7mZunL+C250s1b10AFbpIWMpL78pfp4zj5rMG8PKCCr76wEcsrazzOpZ4TIUuEqZioqO49SsDeW7KePYdaOaSabP5y9wNuhgpgqnQRcLcyQXp/O3mUxhbkM5dLy/mh39dyN6GJq9jiQdU6CI+0D0pnmeuO5lbvzKQV0srufTh2brCNAKp0EV8IirKuPmsATx17Rg2bd/DhQ9+zEerdIuNSKJCF/GZMwb14PXvn0JWcgKTn5zHU5+s1bh6hFChi/hQn+6JzLhxAmcXZvGL15dx9ytLONDU7HUsaWcqdBGfSoyP4ZFvj2bq6f3489wNXPvUPOr2HvA6lrQjFbqIj0VFGXdMGsz9lw9j3tptXPnIHCp37PU6lrQTFbpIBLiiOI+nrzuZyh17uXTabJZvOfwO2OIHKnSRCDGxfwbPTx2Pw3HFw3OYW17rdSQJMRW6SAQp7JnCyzdOpEdKPNc8OY/3l1d5HUlCSIUuEmF6pXbh+e+NZ2BWMlOenc+rCzd5HUlCRIUuEoG6J8Xzl++OZVSfNG7560Kmz9vgdSQJARW6SIRKTojl2e+czGkDMrlzxmL+9Ol6ryNJG6nQRSJYQmw0j10zmrMG9+DuV5bwzOx1XkeSNlChi0S4+JhoHv72aL5SlMXPXluqUg9jKnQRIS4mimlXjeKcYKk/pzH1sKRCFxEAYqOj+MO3RnL6wEzufHkxLy+o8DqSHCcVuogcFB8TzaNXj2Z83+7c9nwpf1+82etIchxU6CLyJQmx0Tw+uZiRvdP4wXMLmb16q9eRpJVU6CLyT7rGxfDk5DEUZCTy3WdLWFSxw+tI0goqdBE5om5dY3n2+pNJS4zj2qc+Y03NLq8jyTEcs9DN7EkzqzazJS183s3MXjezUjNbambXhT6miHghKyWBP14/liiDa5+aR039fq8jyVG05gj9aeC8o3x+E7DMOTccOAP4jZnFtT2aiHQGBRmJPDF5DDX1+7nhmc/Y09DodSRpwTEL3Tk3C9h2tEWAZDMzICm4rPa4iI8Mz0vlD98cxeJNddw8fSFNzXpGaWcUijH0B4FCoBJYDPzAOaeHF4r4zFeKsvjZhUN4t6yK//e3ZV7HkSMIRaGfCywEegEjgAfNLOVIC5rZFDMrMbOSmpqaEKxaRDrS5An5fGdiAU99so6/zNXVpJ1NKAr9OmCGC1gNrAUGH2lB59xjzrli51xxZmZmCFYtIh3trvMHc/rATH766hJmr9Ec9c4kFIW+ATgLwMyygEFAeQi+V0Q6oZjgLQLyMxK58c+fs27rbq8jSVBrpi1OB+YAg8yswsyuN7OpZjY1uMh/ARPMbDHwHvAj55z+ty3iYykJsTwxuRiA7z5bwq79mgfRGZhz3pytLi4udiUlJZ6sW0RC45PVW7n6ibmcOySbaVeNIjDZTdqTmc13zhUf6TNdKSoiJ2xi/wzunFTI35ds4eEP13gdJ+Kp0EWkTW44tYALh/fi/rdX8MGKaq/jRDQVuoi0iZlx72VDGZSVzC1/XcimHXu9jhSxVOgi0mZd42J4+NujaWxy3PTnz2lo1LWFXlChi0hIFGQkcv/lw1i4cQf3vFnmdZyIpEIXkZCZNLQn103M5+nZ6/jbIj3tqKOp0EUkpO6cVMjI3qn86KVFbKjd43WciKJCF5GQiouJ4oFvjCTK4PvTNZ7ekVToIhJyeeldufeyYZRW1PHrf6zwOk7EUKGLSLuYNLQn3x7Xm8dmlTNT89M7hApdRNrN3RcUMTg7mdufL9Xj6zqACl1E2k1CbDR/+OZIdu1v5N9fLMWre0dFChW6iLSrAVnJ3HV+IR+sqOHZOeu9juNrKnQRaXfXjO/DmYMyuefNMlZV1Xsdx7dU6CLS7syM+y4fTlJ8DDc/t5D9jU1eR/IlFbqIdIjM5HjuvWwYZZt38vt3V3kdx5dU6CLSYc4uyuLrxXk88uEa5q/f5nUc31Ghi0iH+smFRfRK7cKtz5eyW4+uCykVuoh0qKT4GH5zxXA2bNujuzKGmApdRDrc2L7d+e6pffnz3A3MWlnjdRzfUKGLiCdu/cpA+vdI4o6XFrFz3wGv4/iCCl1EPJEQG82vrxjOlp37+OUbGnoJBRW6iHhmRF4q3zu9H38t2agbeIWACl1EPHXL2QMYmBUYeqnbq6GXtlChi4in4mMCQy819fu5528aemmLYxa6mT1pZtVmtuQoy5xhZgvNbKmZfRjaiCLid8NyU5lyWmDoRbNeTlxrjtCfBs5r6UMzSwWmARc554YAV4QmmohEklvOHkDfzETunLGYXbrg6IQcs9Cdc7OAo12j+y1ghnNuQ3B5ndkQkeOWEBvN/ZcPo7JuL/f+fbnXccJSKMbQBwJpZvaBmc03s2taWtDMpphZiZmV1NTor1Ui8mWj+6TznYkF/PHT9XxaXut1nLATikKPAUYDFwDnAj8xs4FHWtA595hzrtg5V5yZmRmCVYuI39x+ziB6p3fljpcWse+AbrN7PEJR6BXAW8653c65rcAsYHgIvldEIlCXuGj++9KhrKvdw2/fXel1nLASikJ/FTjVzGLMrCswFtDcIxE5YRP6Z/CNMXn876xyFlXs8DpO2GjNtMXpwBxgkJlVmNn1ZjbVzKYCOOfKgLeARcA84HHnXItTHEVEWuPO8wvJSIrnP15cxIGmZq/jhAXz6incxcXFrqSkxJN1i0h4eHvpFr73x/n8+7mDuOnM/l7H6RTMbL5zrvhIn+lKURHptM4dks35Q7P5/XurWFOzy+s4nZ4KXUQ6tZ9fNISEmCjunLGY5mZvRhTChQpdRDq1HskJ3H1BEfPWbuO5zzZ6HadTU6GLSKd3RXEuE/p151dvllG1c5/XcTotFbqIdHpmxj2XDKWhqZmfvqpJdC1RoYtIWMjPSOSWswfy9tIq3lqyxes4nZIKXUTCxg2nFlDUM4WfvrpED8M4AhW6iISN2Ogo7r1sGFt37efet3RHxsOp0EUkrAzN7cZ3Jhbwl7kbmLf2aHf2jjwqdBEJO7eeM5DctC7cMUN3ZDyUCl1Ewk7XuBjuuWQo5TW7mTZztddxOg0VuoiEpdMGZnLpyBymfbCGFVvqvY7TKajQRSRs3f3VIlK6xPKjlxbRpNsCqNBFJHylJ8bx068WsXDjDp6ds87rOJ5ToYtIWLt4RC9OH5jJ/W+vYNOOvV7H8ZQKXUTCmpnxy0tOAuDulxfj1TMeOgMVuoiEvdy0rtx+ziBmrqjhtdJKr+N4RoUuIr4weUI+I/JS+cXry9i2u8HrOJ5QoYuIL0RHGfdeNoydew/wn68v9TqOJ1ToIuIbg7KTufHM/ryysJKZK6q9jtPhVOgi4is3ndmPAT2S+PGMxeza3+h1nA6lQhcRX4mPiebey4exeec+7ouwOzKq0EXEd0b1TuPaCfk8O2d9RN2RUYUuIr50+zmDAndkfCly7sioQhcRX0qMj+Hey4ZRvnU3v313pddxOsQxC93MnjSzajM76pNZzWyMmTWZ2eWhiycicuIm9s/gG2Py+N9Z5ZRu3OF1nHbXmiP0p4HzjraAmUUD9wJvhyCTiEjI3HVBIT2SE/iPFxfR0NjsdZx2dcxCd87NAo51VuH7wEtA5E38FJFOLSUhll9echIrqup50OcPw2jzGLqZ5QCXAI+0YtkpZlZiZiU1NTVtXbWISKucVZjFJSNzmDZzNUsr67yO025CcVL0d8CPnHPHPI3snHvMOVfsnCvOzMwMwapFRFrnZxcWkZYYx23Pl/p26CUUhV4MPGdm64DLgWlm9rUQfK+ISMikdo3jnkuGsnxLPQ/5dOilzYXunCtwzuU75/KBF4EbnXOvtDmZiEiIfaUoMPTykE+HXlozbXE6MAcYZGYVZna9mU01s6ntH09EJLT8PPQSc6wFnHPfbO2XOeeubVMaEZF2lto1jl9dMpQbni3hgfdWcfu5g7yOFDK6UlREIs7ZRVlcMTqXaR+sZsGG7V7HCRkVuohEpJ9cWER2SgK3vVDqm3u9qNBFJCKlJMRy3+XDKa/ZzX1vrfA6Tkio0EUkYp0yIINrxvfhyU/WMnv1Vq/jtJkKXUQi2p2TCumbkcjtL5RSt/eA13HaRIUuIhGtS1w0v/36CKrq9/Pz18L74dIqdBGJeMPzUvn+v/Tn5QWbeGNRpddxTpgKXUQEuOnM/gzPS+XHLy9hc91er+OcEBW6iAgQGx3F774+ggNNzdz2fCnNzc7rSMdNhS4iElSQkcjPLixi9ppa/vejcq/jHDcVuojIIa4szuO8Idn8+h8rWLIpvG7gpUIXETmEmfGrS4eSnhjHzc8tYE9Do9eRWk2FLiJymLTEOH575QjWbt0dVlMZVegiIkcwoX8GN57Rj+dLKnitNDymMqrQRURacMvZAxnVO5W7ZixmQ+0er+MckwpdRKQFsdFR/P4bIzGD7z+3oNM/EEOFLiJyFHnpXbnvsmGUbtzBfW8t9zrOUanQRUSOYdLQnkwe34fHP17LP5Zu8TpOi1ToIiKtcNcFhZyUk8LtL5SycVvnHE9XoYuItEJ8TDQPfWsUzsG/Te+c4+kqdBGRVurTPZH7Lg+Mp9/zZpnXcf6JCl1E5DhMGtqT608p4OnZ63h14Sav43yJCl1E5DjdMWkwxX3SuOOlxaysqvc6zkEqdBGR4xQbHcVDV40iMT6GqX+aT/2+zvHoOhW6iMgJyEpJ4MFvjWR97Z5Oc//0Yxa6mT1pZtVmtqSFz68ys0XBn9lmNjz0MUVEOp9xfbtz1/mF/GNZFdM+WO11nFYdoT8NnHeUz9cCpzvnhgH/BTwWglwiImHhOxPz+dqIXvzmnZXMXF7taZZjFrpzbhaw7Sifz3bObQ++/BTIDVE2EZFOL3D/9GEUZqdw83MLWLt1t2dZQj2Gfj3w95Y+NLMpZlZiZiU1NTUhXrWIiDe6xEXz6NWjiY2O4oZnPmOnRydJQ1boZnYmgUL/UUvLOOcec84VO+eKMzMzQ7VqERHP5aV3ZdpVo1hfu4ebpy+gyYOTpCEpdDMbBjwOXOycqw3Fd4qIhJtxfbvzi4uH8MGKGk/uzBjT1i8ws97ADOBq59zKtkcSEQlfV43tw4ot9Tw6q5z+PZK4ojivw9Z9zEI3s+nAGUCGmVUAPwNiAZxzjwA/BboD08wMoNE5V9xegUVEOruffLWINTW7uOvlxeSld2Vc3+4dsl5zzpvJ8MXFxa6kpMSTdYuItLe6PQe49OFPqN3dwMs3TqQgIzEk32tm81s6aNaVoiIi7aBb11ievHYMBlz/9Gfs2NPQ7utUoYuItJM+3RN57JpiKrbvZcof57O/sald16dCFxFpR2Py0/n1lcOZt3Zbu9/zpc2zXERE5OguGt6Lyh17+e+/LycnrQt3Tipsl/Wo0EVEOsD3TutLxfY9PPphOTmpXbhmfH7I16FCFxHpAGbGzy8cQt3eRnJSu7TLOlToIiIdJCY6ij98c2S7fb9OioqI+IQKXUTEJ1ToIiI+oUIXEfEJFbqIiE+o0EVEfEKFLiLiEyp0ERGf8Ox+6GZWA6w/wX88A9gawjjhIhK3OxK3GSJzuyNxm+H4t7uPc+6ID2X2rNDbwsxKIvGpSJG43ZG4zRCZ2x2J2wyh3W4NuYiI+IQKXUTEJ8K10B/zOoBHInG7I3GbITK3OxK3GUK43WE5hi4iIv8sXI/QRUTkMCp0ERGfCLtCN7PzzGyFma02szu8ztMezCzPzGaaWZmZLTWzHwTfTzezd8xsVfDXNK+ztgczizazBWb2RvB1gZnNDW73X80szuuMoWRmqWb2opktD+7z8ZGwr83sh8E/30vMbLqZJfhxX5vZk2ZWbWZLDnnviPvXAh4I9tsiMxt1POsKq0I3s2jgIWASUAR808yKvE3VLhqB25xzhcA44Kbgdt4BvOecGwC8F3ztRz8Ayg55fS/w2+B2bweu9yRV+/k98JZzbjAwnMC2+3pfm1kOcDNQ7Jw7CYgGvoE/9/XTwHmHvdfS/p0EDAj+TAEePp4VhVWhAycDq51z5c65BuA54GKPM4Wcc26zc+7z4O/rCfwHnkNgW58JLvYM8DVvErYfM8sFLgAeD7424F+AF4OL+Gq7zSwFOA14AsA51+Cc20EE7GsCj8DsYmYxQFdgMz7c1865WcC2w95uaf9eDDzrAj4FUs2sZ2vXFW6FngNsPOR1RfA93zKzfGAkMBfIcs5thkDpAz28S9Zufgf8B9AcfN0d2OGcawy+9ts+7wvUAE8Fh5keN7NEfL6vnXObgF8DGwgUeR0wH3/v60O1tH/b1HHhVuh2hPd8O+/SzJKAl4BbnHM7vc7T3szsq0C1c27+oW8fYVE/7fMYYBTwsHNuJLAbnw2vHElwzPhioADoBSQSGG44nJ/2dWu06c97uBV6BZB3yOtcoNKjLO3KzGIJlPmfnXMzgm9XffHXr+Cv1V7laycTgYvMbB2B4bR/IXDEnhr8azn4b59XABXOubnB1y8SKHi/7+uzgbXOuRrn3AFgBjABf+/rQ7W0f9vUceFW6J8BA4JnwuMInER5zeNMIRccN34CKC/huFsAAAEESURBVHPO/c8hH70GTA7+fjLwakdna0/OuTudc7nOuXwC+/Z959xVwEzg8uBivtpu59wWYKOZDQq+dRawDJ/vawJDLePMrGvwz/sX2+3bfX2Ylvbva8A1wdku44C6L4ZmWsU5F1Y/wPnASmAN8GOv87TTNp5C4K9Zi4CFwZ/zCYwnvwesCv6a7nXWdvx3cAbwRvD3fYF5wGrgBSDe63wh3tYRQElwf78CpEXCvgZ+ASwHlgB/BOL9uK+B6QTOExwgcAR+fUv7l8CQy0PBfltMYBZQq9elS/9FRHwi3IZcRESkBSp0ERGfUKGLiPiECl1ExCdU6CIiPqFCFxHxCRW6iIhP/H9w6IpeIwAr8gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores3)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD6CAYAAACxrrxPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3xUVfrH8c+TSUJCCITeQggloPQSqg0FAVFBLFTFutiw7c+21rWXtSy7iwURRUSCYkMFEQuKChh6CS2EklBDSSN95vz+uJMQwkCGkGRKnvfrxYuZe8/MPJchXw7nnnuuGGNQSinl+wI8XYBSSqmKoYGulFJ+QgNdKaX8hAa6Ukr5CQ10pZTyExroSinlJ9wKdBEZKiJbRCRRRB51sb+liPwkIutEZLGIRFZ8qUoppU5HypqHLiI2YCtwKZACxANjjTEJJdp8BnxrjJkhIpcANxtjbjjd+zZo0MBER0efZflKKVW9rFy58pAxpqGrfYFuvL43kGiMSQIQkThgBJBQok0H4AHn41+Ar8p60+joaFasWOHGxyullCoiIrtOtc+dIZfmQHKJ5ynObSWtBa5xPh4JhItIfReFTBSRFSKyIjU11Y2PVkop5S53Al1cbCs9TvMgcJGIrAYuAvYAhSe9yJipxphYY0xsw4Yu/8eglFKqnNwZckkBWpR4HgnsLdnAGLMXuBpARGoB1xhj0iuqSKWUUmVzp4ceD8SISCsRCQbGAPNKNhCRBiJS9F7/AKZXbJlKKaXKUmagG2MKgUnAQmAT8KkxZqOIPCsiw53NBgBbRGQr0Bh4oZLqVUopdQplTlusLLGxsUZnuSil1JkRkZXGmFhX+/RKUaWU8hMa6EopVVUcDlj4OBzaVilvr4GulFJVZV0cLP0fJP9VKW+vga6UUlUhJw0WPQWRvaDr2Er5CHfmoSullDpbi1+GY4dg/FwIqJy+tPbQlVKqsh3YCH9NhdiboVm3SvsYDXSllKpMDgfMfxhCasMlT1bqR+mQi1JKVZb8bPjqTtj1O1w5GWrWq9SP00BXSqnKkLkfZo+Fvavh0mehx42V/pEa6Eop5UrC17D8XSjv1fSHt1k99DGz4JzLK7a2U9BAV0qp0jL3w1d3Qc36EBFVvvdo3hMufhyadqnY2k5DA10ppUpb9BTY8+GGL6F+G09X4zad5aKUUiXt+hPWzYH+9/pUmIMGulJKHWcvhPkPQZ0WcMH/ebqaM6ZDLkqp6m3tHNi91HqcuR8ObIBRH0FwTc/WVQ4a6Eqp6sleCN8/CvHvQWhdCAiytne/Ac4dfvrXnqHcAjuZuYXk5NvJKbDTMLwG9cKCK/QzQANdKeXPjAHjOHl7bjp8fits/xn63wODnoEAW4V85JFj+Ww9kMmW/ZmsTUljbXIa21OPndDmhZGdGN+nZYV8XkluBbqIDAUmAzZgmjHm5VL7o4AZQISzzaPGmPkVXKtSSrkvOR4+uxEy9rjeHxAIw/8LPSac9Uc5HIZZy3fx358TOZiZV7y9Qa0adGtRhyu7NqN+WDChwYGEBtno3LzOWX+mK2UGuojYgCnApUAKEC8i84wxCSWaPYF1r9G3RaQDMB+IroR6lVKqbOvnWvPIazeFAY+ByMlt2gyEyJ5n/VHJR7J55PN1/Ln9MP1a12fiha2JaRxOu8a1aFI7BHH12ZXEnR56byDRGJMEICJxwAigZKAboLbzcR1gb0UWqZRSbnE44NeX4ddXoOV5MGomhNWv8I9JOZrN0u2HWZZ0hO837APgpas7M6ZXiyoN8NLcCfTmQHKJ5ylAn1Jt/gn8ICL3AGHAIFdvJCITgYkAUVHlvPpKKaVcKcixFsLa+CV0ux6ueBMCK+7E49Fj+Xy9Zg+frUxh494MAOqFBTOoQ2MeGtKeyLqenxXjTqC7+uem9OIGY4EPjTGvi0g/YKaIdDLmxLMRxpipwFSA2NjYci6QoJRSpZReCKv/va6HWVw4eiyfQoehYXiNE7YnHsxi8ZaDbDuQxdaDmWzck0G+3UHHZrV54vJzuSCmITGNahEQ4LkeeWnuBHoK0KLE80hOHlK5FRgKYIxZKiIhQAPgYEUUqZRSJ8jYawV4fpb1PCsVHIVnvBBWZm4BI6b8wZ60HIZ0bMwNfaMJDhTeXpzEj5sOAFYvPKZRLW7s35KR3SPp0Kx2Ge/qOe4EejwQIyKtgD3AGGBcqTa7gYHAhyJyLhACpFZkoUopVWzFB7BvLXQcafXEbcHQ725o0vmM3uaZbxJIOZrNqNgWLNiwn/nr9wMQUTOIewfGMK53FE3qhFTGEVSKMgPdGFMoIpOAhVhTEqcbYzaKyLPACmPMPOD/gPdE5AGs4ZibjCnvmpNKKXUa9kJY/TG0HQjXfVDut1mwfh9zV6ZwzyVt+b/B7Xn6yo58t34f+YUOrurejJrBvneZjlsVO+eUzy+17akSjxOA8yq2NKWUciFxEWTuhWGvlvstDmTk8o8v19Mlsg73DowBIDTYxrU9IyuqSo/wvX+ClFLV28oZUKsxtBtavpfvOsIz3ySQW2DnzdHdCLL5zxqFGuhKKd+Rvge2LYTz7gdbkNsvM8YQv/Mo//15G0u2HaJBrWDeGNWNNg1rVWKxVU8DXSnlO9bMstZmceNyfWMMKUdz+GbdXuauTCEp9Rj1woL5x2XncEO/lj45Rl4W/zsipZR/cthh1UfQegDUa+WySXpOAf/9aRurk9PYeiCTzNxCAHpF1+WOC9tweZemhNXw39jz3yNTSvkPhwN+egbSk2Hwcy6bLE86zN8/XcuBjFx6RNVleNdmtGsczoXtGtKqQVgVF+wZGuhKKe+Wfwy+vB02fQM9bjxprfJCu4N//7iNtxYnElWvJp/f2Z+uLSI8VKxnaaArpbxXVirMugb2r4chL0HfO0+4pH9/ei73zl7NXzuPMCo2kqev7OjXQyplqb5HrpTyfgv/AQc3wdg4aDfkhF2/bU3lgTlryCmw8+/R3biqe3MPFek9NNCVUt5p5x+w/jO48OETwjwjt4B/fb+Fj5fvIqZRLd4a35O2jfxr+mF5aaArpbyPvRDmPwR1ouD8BwBrGuL3G/bz9LyNHMrK46b+0Tw0pL1fTj8sL/2TUEp5n/hpcHAjjP4YgmuyLiWNl+ZvZmnSYTo0rc20G2PpElk9T3yejga6Usq7ZB2EX16ENpdwOPJSnpm9mnlr91IvLJh/XtmB6/u2JNCPLtevSBroSinvcXg7zB4DhTkcuuA5xry3nOQj2dxzSVsmXtia8BD3L/evjjTQlVLeYefvMOd6AFJHxnHdZ6mkZubx0S296dO64u8L6o/0/y1KKc9bNRM+ugrCGrL3uvlc9a1wOCufj27to2F+BjTQlVKe47DDD0/AvEkQfT7p4xcw4atUMnML+ORvfenZsq6nK/QpOuSilKo6BTlwaKv12Dhg8SuwdQH0uo2CwS9x94zV7Dx0jJm39qFzZB3P1uqD3Ap0ERkKTMa6Bd00Y8zLpfa/CVzsfFoTaGSM0TlFSqnjDm2DT0bBkaTj28QGw17D9LqNp7/awO+Jh3j12i70a6PDLOVRZqCLiA2YAlwKpADxIjLPeds5AIwxD5Rofw/QvRJqVUr5qu2/wGc3QkAQXPUOhNS2ttdtxe7AaP796Vq+WL2HOwe0YVRsC8/W6sPc6aH3BhKNMUkAIhIHjAASTtF+LPB0xZSnlPJ58e9bV302bG+tyVK3JWAtrPXmoq18vmoxAQHCHRe14aHB7T1crG9zJ9CbA8klnqcAfVw1FJGWQCvg51PsnwhMBIiKijqjQpVSPsZeCD88DsvfgZjBcM37xT3zpdsPM+mTVWTmFXJ935bcOaANjWuHeLhg3+dOoIuLbeYUbccAc40xdlc7jTFTgakAsbGxp3oPpZSvy02HubdC4iLoexcMfh4CbBhj+OCPnbwwfxPR9Wsy5/Z+urBWBXIn0FOAkoNakcDeU7QdA9x9tkUppXzY0Z3wyRg4vA2u+DfE3gxYN6J4/MsNzFmRzOAOjXl9VFe98rOCuRPo8UCMiLQC9mCF9rjSjUSkPVAXWFqhFSqlfMeupTBnPDgK4fovoPVFAOQW2Lln9moWJRzg3kvacv+gdgQEuPrPvzobZQa6MaZQRCYBC7GmLU43xmwUkWeBFcaYec6mY4E4Y4wOpShVHa2Ng3n3QJ0WMO5TaNAWgMzcAv720QqWJR3hmeEdubF/tGfr9GNuzUM3xswH5pfa9lSp5/+suLKUUj7D4YCfn4Pf34DoC2DUR1CzHgDJR7L520crSDyYxeQx3RjRTe8qVJn0SlGlVPnlH4MvJsLmb60bOF/+OtiscfEl21K5Z/ZqHA7D9Jt6cWG7hh4u1v9poCul3Hd4uzWskuGcF5GXATlHT7iBs91heOfX7bz+wxZiGoUzdUJPWtYP82zd1YQGulLKPTuWwKc3AAJtB1nbRKDLaGg7EIDtqVk8+NlaVu9O44ouTXnlmi6E1dCYqSr6J62UKtuqmfDt/VCvDYyLg3qtT9idlVfIR0t3MvnHbYQE2Zg8phvDuzZDRGeyVCUNdKXU6e1YYi1v2+YSuO5DCDm+CmJqZh4f/rmDmUt3kZFbyKUdGvPCVZ1opFd9eoQGulLq1OwFsOBhiIiCMZ9AUChgzSt///cd/O/nRHIL7VzWqQkTL2xDtxa6yKonaaArpU4tfhocTIDRsyAoFGMMixIO8ML8Tew6nM1lnZrw8NBzaNVAT3p6Aw10pZRrWQfhlxeh7SAKYi7jm1UpTP0tic37M2nbqBYf39qH82MaeLpKVYIGulK+zl5o3f0nMPjE7cY4pxe6uHi7RvgJY+HFjh2GwhwAChc9Q0BBDv8OvIU5r/7CgYw82jWuxevXdWV4t2YE2fQOlt5GA10pX3Z0J8weC3mZ1lrjTTpZ27OPwGc3wY5fXb/OFgxXvAndr7eeOxxkLnyW8OVvFjcJBN4qHM60BBvntY1gbO8WDGjXSNdg8WIa6Er5qt3LIG6ctRBWUE14fzBc+z7Uj7Fu9ZaeDBc/AeGNT37ths/h67tJ27WeN/JHMmDLM1xi/5Mv7OezSjrSvG4oTRo0oHOva1jdtjE1Am1Vf3zqjImn1tKKjY01K1as8MhnK+XzNnwOX97hXAhrDgTXgtljYN9a63FgsHUis2U/ly/Py8tl84eT6LrvM9JNGOGSzYqY+wkbcD/tm9QmUIdTvJaIrDTGxLrapz10pXxN+h74ehI06wFjZxcvhMXNC+Cbe+FwojVfvG508UsOZuSyaX8mKUezSTmaww8b97M9dST/imrK1blfEjBsGr3PGeaRw1EVRwNdKV/zw+PWSdCrpx4Pc4DgmnDNNDJzC1ifks7atdtZm5zG2pQ09qXnFjcLsgltGtbig5t7cXH7y4Hnqv4YVKXQQFfKlyT9Chu/hAGPFd9sGWBPWg4L1u/j23X7WJuSRtFIasv6NekVXY+uLSLo2Kw2LevXpFF4CDY9semXNNCV8hXFV222JCv2LpYlHGBZ0mGWJh1m494MADo1r839A9vRLSqCLs3rUDcsuIw3Vf5EA10pb7Z/PezfYD3eswJSNzMz+mVeevUPsvPtBAcG0CMqgoeGtOfyzk2J1is2qzW3Al1EhgKTsW5BN80Y87KLNqOAf2JdxbDWGHPSfUeVUmfg8HZ47xKw5xdvWmDvxXOJLRnetSlX92hOj6i6hATplEJlKTPQRcQGTAEuBVKAeBGZZ4xJKNEmBvgHcJ4x5qiINKqsgpWqFoyBBY/gCAjmtRZT+GZLFhGhwQy7qA/Lerekng6lKBfc6aH3BhKNMUkAIhIHjAASSrT5GzDFGHMUwBhzsKILVapa2bIAEhfxLzOBD7aHc9uArtx+UWvCQ4I8XZnyYu4EenMgucTzFKBPqTbtAETkD6xhmX8aY74v/UYiMhGYCBAVFVWeepXyfwU5ZM97kD0mkl9qX8WPt/Qjsm5NT1elfIA7ge5qflPpy0sDgRhgABAJLBGRTsaYtBNeZMxUYCpYV4qecbVK+ZMjSZBzwo8IBtj64we0z97DjIiXmD3xAp2potzmTqCnAC1KPI8E9rpos8wYUwDsEJEtWAEfXyFVKuVPHA5Y/BL89upJuwRoDywNvYh/3DVR78epzog7f1vigRgRaQXsAcYApWewfAWMBT4UkQZYQzBJFVmoUn4hPxu+uhMSvoJu4+Hc4WTlFTJ//T5+3HSAIFsAV3aPYuCwawkK1jBXZ6bMvzHGmEIRmQQsxBofn26M2SgizwIrjDHznPsGi0gCYAceMsYcrszClfI5mfutpW73roZLnyOt2+289/sOPvxjJ9kFzbiuZ28eHnoODWrV8HSlykfpaotKVYV9a+GTMThy0/ijy0vMPNqJJdsOkVtoZ1jnptw/MIaYxuGerlL5AF1tUSlP2vQN5ouJZBDO2GNPkvBHQ5rVSefqHs2Z0C+a9k00yFXF0EBXqjJtWwRzrmejxHBz7gMM7t2FN/q1pH3jcER0gSxVsTTQlaokuTnHyPrsPtIczXg04kXeuakXPVvW9XRZyo9poCtVCZJSs1jy/iPcmL+Hnzr8hy+uHURwoN4FSFUuDXSlKojDYVidnMZ36/bxW/xKvgn4lIORQxg95kZPl6aqCQ10pc7SsbxCPlq6i5lLd7I3PZdgWwBxEZ9SI89G6LWve7o8VY1ooCtVTtn5hcxcuot3f93OFXnf8mXIImrVhdCgAAIyUuCSJyCiRdlvpFQF0UBX6gzl5NuZtXwX7/y6nbSsbN6t/ykDHd9C835Qt5XVqHZT6H+vZwtV1Y4GulJuOnIsnznxybz/+w4OZeUxuHUNXmv4LrX3/QHn3Q8Dn4YAPfGpPEcDXakybNiTzvQ/dvDtun3kFzo4r2193u9Vl66/3Q5Hd8KIt6D7eE+XqZQGulKuGGP4PfEQ7/y6nT8SDxMWbGN0bAtu6NeSdtlrYM5okACY8DVEn+fpcpUCNNCVOknSwQz++OhpItNXcldgAC81q0nTiFCCsgR+cMCOX6FeGxg3B+q18nS5ShXTQFfKyeEwzFyyicY/388Nspy0Om2pXbs2AeRAbs7xhp2uhWGvQkgdzxWrlAsa6EoBeYV2Hp6+kFtSHqdzwA4yL3qGiAH3ga63onyIBrqq9uwOw+sffsqjex6jQVAOct0nhJ8zzNNlKXXGNNBVtWaMIW7GFO5PfhZHSF2Cbp4HTTp7uiylysWtQBeRocBkrDsWTTPGvFxq/03Av7BuUQfwP2PMtAqsU6kzU5ADy9+BbOeNs8QG3cZBw/bFTQoL7Sz58AnGp7zF3vCONLv9Swhv7KGClTp7ZQa6iNiAKcClWDeDjheRecaYhFJN5xhjJlVCjUqdmcz9EDcO9qyEoJrWNns+xL8P106HdoNZt/MABz+5g0H5P7M2YiBd7voYgmt6tm6lzpI7PfTeQKIxJglAROKAEUDpQFfK8/atg9ljIOcojJ4F515hbU/fA7PHYGaP5qemE4lI+ZlBAVvZ1uEeul73nJ78VH7BnUBvDiSXeJ4C9HHR7hoRuRDYCjxgjEl20UapypO6BaYPhdAIuGUhNO1SvCu3ZhNmxrxNq/0PMGjvOxTYgskePo2Y7td5sGClKpY7ge6q61L6ztLfALONMXkicgcwA7jkpDcSmQhMBIiKijrDUpU6DWNg/kNgC4TbfoTazYp3rdx1hHtnr2FPWg6D2j9P5+jVND6nH0FNu3qwYKUqnjuBngKUXAM0EthbsoEx5nCJp+8Br7h6I2PMVGAqQGxsbOl/FJQqv4SvrSs4h71WHObGGKYt2cEr32+mWUQos//Wl35t6uP6P5hK+T53Aj0eiBGRVlizWMYA40o2EJGmxph9zqfDgU0VWqVSp5N/DBY+Do07Q8+bAUjPLuDBuWtZlHCAoR2b8Op1XagdEuThQpWqXGUGujGmUEQmAQuxpi1ON8ZsFJFngRXGmHnAvSIyHCgEjgA3VWLNSp1oyeuQkQLXvg+2QNYmp3H3J6s4kJHLk1d04JbzohE96amqATHGMyMfsbGxZsWKFR75bOUjdv4B8dNgyIvWDSNc2b0MZlwJHa/GjHyHGX/u5IX5m2gUHsKU8T3o1iKiamtWqpKJyEpjTKyrfXqlqPJePz8Pu/+E3UthbBw063bi/rVzYN4kqBPJrh6P8Mh7y1iWdISB5zTi9VFdiagZ7Jm6lfIQDXTlnVK3WGHe/QZIWmxNR7z8NWh4jrV/87fw+5sURp3PtKbP8Ma0LYQEBvDS1Z0ZHduCgAAdYlHVjwa68k4rZ0BAkHVbt4FPWVd+fn33CU3+rHM5E5PGkrX1AFd2bcaTV5xLo/AQDxWslOdpoCvvU5ALa2fDOcOgVkNr203fwa4/2Zmazqxlu1h5wEFi+rmMjI1kVGwLOkfq2uRKaaAr77P5W8g5Aj1vKt6Unh/AK+saMPuvbOrV7MCDI9szsntzQoJsnqtTKS+jga68z8oPIaIltBoAwLKkw/x9zhoOZOZxU/9o7h/UjjqhOqdcqdI00JV3OZQIO5fAJU+SU2j43y+beWvxdqLrh/HlXf3pEqnTEJU6FQ105XHH8grZdTibo2lHaLv4Phpi4/b157B44UIK7IbRsS146soOhNXQv65KnY7+hKgqVWB3sGV/JmuS01ibnMbalDS2HcyiqTnE+8GvUV9SeLLwZg5Sl1vOr8fF7RvRt3V9T5etlE/QQFeVyhhD4sEsftp8kANrFxGWuga7w7o6uUWwjd51QmgWE0zsgc+wOfI4MORjHus8RHvjSpWD/tSoCmeMYePeDL5bv4/56/ex6/Ax7rB9w9NBcdZqQEUTUwyQ5vxVvy2MnkXzRud4rG6lfJ0GuqoQBzJy+XP7IZZuP8yf2w+TcjQHW4BwUZvaTI+YQ5s986DTtXDFG2CrcfIb2IIhIKDqC1fKj2igK/el74E510PWAQDsDkNugZ2cAjt2u4M+QD8RHrUFUCMigJAgG7ZDudac8gGPwUUP663elKpEGujKfT88gTmYwO5ml7HzcDb7M/LAQETNIFo0DKVpRCgRoUEnZ3b7YXDO5R4pWanqRANduWfHb7DxC96VUby89SqaR4RyzYWRjOjWjDYNa3m6OqUUGujKHfYCmP8Q+eFRvJk6jMeGncNt57fWFQ2V8jJ6FkqdpNDuIPlI9vENf02F1M0k93mSPIJp1zhcw1wpL+RWoIvIUBHZIiKJIvLoadpdKyJGRFzeTUN5t5x8Ox8t3cnFry/mgld/YcehY9a65L+8BDGDOdD4YgBdEEspL1XmkIuI2IApwKVAChAvIvOMMQml2oUD9wLLK6NQVfHWpaQxd2UKh7LyOJyVz5YDmaRlF9A8IhSAvM2LYMm9EBQCl71C7kEHAKEa6Ep5JXfG0HsDicaYJAARiQNGAAml2j0HvAo8WKEVqgqXlVfIawu38NHSnYQE2WhaJ4T6YTW4uH0jxvaOwhYAX019hvY/zbTuEDQuDiKiyEnZB2gPXSlv5U6gNweSSzxPAfqUbCAi3YEWxphvRUQD3UvZHYZv1+3l5QWb2Z+Ryw19W/LgkPbUDjlxKdqU32bwXNCHHGxyMY1umgk1wgHIKbAD2kNXylu5E+iuzn6Z4p0iAcCbwE1lvpHIRGAiQFRUlHsVqrPmcBi+W7+PyT9tI/FgFuc2rc2U8T3oEVX35Ma5GTRZ9jxrHK3Z1fs/jHCGOUCuM9BDgvRculLeyJ1ATwFalHgeCewt8Twc6AQsFuuKkibAPBEZboxZUfKNjDFTgakAsbGxBlWp0rML+GxlMjOX7WLX4WxiGtViyrgeXNapyalnqfz6CrbsVJ4quIdxBSfuKg70YO2hK+WN3An0eCBGRFoBe4AxwLiincaYdKBB0XMRWQw8WDrMVdXJLbDz5qKtzFi6k9wCB7Et6/LQkPZc1qkpttNNNzy4CZa9TX6X61n3VxtGOgO85PuCDrko5a3KDHRjTKGITAIWYq2TN90Ys1FEngVWGGPmVXaRyn2rdh/lwU/XknToGFf3aM5t57emQ7PaZb/QGJj/ENQIxwx8Cv6KLx4zL5JTYMcWIATZdMhFKW/k1pWixpj5wPxS2546RdsBZ1+Wcld+oYOEfRmsS0kjfudRvlu3l6Z1Qpl1Wx/Oa9ug7DcAK8x/+5d167fLX6dG7YaIQG5+6R66Q3vnSnkxvfTfRxlj+HrNXl5asIkDGXkANKhVg+v7tuShIe0JD3HzJsqFeTDvXlgXB52vg543IyKEBtlc9tD1hKhS3ksD3Qcl7M3g6XkbiN95lC6RdXjqio50j4qgaZ0Q5EyWpz12COLGQ/IyuPhxuPCh4uVtXQV6br5d56Ar5cU00H3Md+v28cCnawivEcgr13Tmup4tyreuyoEEmD0asg7CtR9Ap6tP2B0SZCMn33HCttxCuw65KOXFNNB9yLQlSbwwfxM9o+oydUIs9cKCy/dGW3+AubdAcE24eT4073lSk9BgW/GsliI52kNXyqtpoPuAAruDF77bxId/7uSyTk14c3Q394PVGPj9Tdj4hfNyMAMHE6BxRxg7B+o0d/kyl0MuelJUKa+mge7l9qfncs/sVcTvPMot57XiicvPdX+IpSAHvr4bNnwOLfpCzfrW9ugL4JInoMapb0wRGuSih15gJzxE/8oo5a30p9OL/bo1lQfmrCG3wM7kMd0Y0c11b7pY9hE4tM167CiARU/DnpUw6Bk4774zup9njaAAsvIKT9iWW2CnUbiLGzwrpbyCBroXSkrN4tXvt/D9xv20bxzOlPE9aNuojNu87V4OceMg+9DxbUE1YfRMOPfKM64hNMhGambeCdtyC3QMXSlvpoHuRXLy7by0YBOzlu8mJDCAv1/ajr9d0JrQstZOWTsH5k2COpEw/D8QGGJtb9AOIlqc/rWn4PKkaIHOclHKm2mge4kDGbncNmMFG/amc32fltw3KIYGtdwY3vjtNfj5OWtcfNRHULNehdRzypOiujCXUl5LA90LbNiTzq0z4snKLeS9G2IZ1KGxey/cuwZ+fh46XQNXvQOB5ZzG6II1D/3kHnoNvVJUKa+lge5Bx/IKmfpbEu/+tp36YTWYe2d/zm3qxkJaAA6HtZhWWAO4/I0KDXMoGnI5fmGRw2HIL9RpizBsyBoAABHDSURBVEp5Mw10D7A7DLP/2s2/f9zGoaw8hnVuwjPDO9HwTGaQrIuDlL9gxFsQGlHhNYYG2ci3Oyi0Owi0BZBbWHRzCw10pbyVBnoVS88p4L641Szekkrv6HpMndDT9Z2DTic3HRY9BZG9oOvYSqmzqCeeW+igli2gePhFe+hKeS8N9Cq0ZX8mt89cwZ60HJ6/qhPj+0S5t5iWMbB+LqTttJ6nrLAW1ho/FwIqZ0y76K5EOfl2atUIJLfQGn7R1RaV8l4a6FVkxc4jTJj+F2E1Apn9t77ERrs5G6UgF765F9bNOXH7+X+HZt0qvlCn4h66c6ZLUQ9dh1yU8l4a6FXgyLF8Jn2ymobhNfj09n40rh3i3guzUmHOeEhebl2q37/E1Z42N9c7L6einnjR1EW9/ZxS3s+tQBeRocBkrFvQTTPGvFxq/x3A3YAdyAImGmMSKrhWn+RwGP7v0zUcOZbPF3f1P32YJ8dbvfHsI9bzvEwwDrhuBnS8qmoKdirdQy++QbQGulJeq8xAFxEbMAW4FEgB4kVkXqnA/sQY846z/XDgDWBoJdTrc6b9nsQvW1J5dkRHOjWvc+qG6+fCV3dBeGNoN8TaFhAIPSZU6tDKqRQFetFQS9EURr2wSCnv5U4PvTeQaIxJAhCROGAEUBzoxpiMEu3DcC7UWt0t2ZbKq99vYWjHJtzQt6XrRsbA4pfg11cgqj+M/hjC6ldtoS4UnxQtGkMv6qEHaqAr5a3cCfTmQHKJ5ylAn9KNRORu4O9AMHCJqzcSkYnARICoqKgzrdWnfLJ8N099vYHWDcN45dourmezFOTAV3fCxi+h23i44k0I9I7VDE86KVo0hh6ss1yU8lbu/HS6mld3Ug/cGDPFGNMGeAR4wtUbGWOmGmNijTGxDRs2PLNKfYTdYXju2wQe+3I957VtwNw7+1Mn1MUJzMz98MEw2PgVXPosjJjiNWEOJYZcdAxdKZ/hTg89BSi5ZF8ksPc07eOAt8+mKF91KCuP++PW8HviIW4+L5rHh51LoM3Fv5lpyTB9COSkwZhZcM7lVV9sGUKL56FbY+ca6Ep5P3cCPR6IEZFWwB5gDDCuZAMRiTHGOO+swOXANqqZ5UmHuWf2atJzCnj1mi6M6nWaZWu/fxRyjsIt30PTrlVX5BkIOUUPXactKuW9ygx0Y0yhiEwCFmJNW5xujNkoIs8CK4wx84BJIjIIKACOAjdWZtHe5vOVKTw0dy3R9cOYcUvv0y+wlfgjbP4WBj7ltWEOri4sKrpSVANdKW/l1jx0Y8x8YH6pbU+VeHxfBdflM5KPZPPk1xvo3aoe027sRa0ap/kjLcyDBY9AvTbQb1LVFVkOQTbBFiDHpy0W2gm2BWBz936mSqkqp1eKngVjDI98vo4AEV4f1e30YQ6w7C04nAjjP/eqE6CuiAghgQHHpy3m61roSnk7DfSz8Mlfu/lz+2FeGNmJ5hGh1sZ9a2FtHDjspVobWD0L2l8OMYOqvNbyCA22nTCGruPnSnk3DfRy2pOWw0vzN9O/TX3G9XbOqd/4JXx5h/U40MUl/hEtYOhLVVfkWQoJsp1w6b9eJaqUd9NAL4f0nALu/HglDmN45Zou1kT9X/8FvzwPLfrA6FlQy/fn2YeWCPScArteJaqUl9NAP0MZuQVMmP4Xm/Zl8Pb4nrSoVxOWvG6FeZfRcOV/IMjN1RS9XGiw7YS1XEK0h66UV9NAPwOZuQVMeP8vEvam89b4ntbNnO0FsPxdaDMQRr57fHlbPxASZDthLZeQQD0pqpQ3059QN9kdhjs/XsWGPelMGdeDSzs0tnZsXQhZB6D33/wqzMEacskpOH6lqI6hK+XdNNDd9O5v2/k98RAvjOzE4I5Nju9Y+SGEN4W2l3qstsoSGmQjN19nuSjlKzTQ3bAmOY03ftjK5V2aMiq2xCX9acnWlZ/dbwCb/41elZy2mFNg16tElfJyGuhlyMwt4N7Zq2lcO4QXR3Y+cRnc1TOt33vc4JniKlnJMfTcAocGulJezv+6lRXIGMOTX20g5Wg2c27vd+IyuPZCWDUT2g6ECP9c2z0kKOD4kEu+vfg+o0op76Q/oacx48+dfLVmL/cPakev6Hon7kz8ETL3Qs+bPFJbVQgt2UMv1DF0pbydBvopLE86zHPfbWLQuY2ZdHHbE3caA39MhlqNoZ3/3jo1NMhGocOQW2CnwG50yEUpL6dDLi7sS8/h7k9W0bJeTd4Y3ZWA0isMrv8Mdv9pXURkc3E3Ij9RNE3xaHa+9VwDXSmvpoFeSqHdwd2zVpGTbyduYl9qh5QK7NwM+OEJaNbDmt3ix4p65EePFVjPdR66Ul5NA72Ud39LYtXuNCaP6UbbRuEnN/j1Fcg6CGNnQ4B/j1gV9ciLeuh6pahS3k0DvYRN+zL494/WfPMR3ZpbGx12696fAEd3wvJ3rGmKzXt6rM6qctKQi/bQlfJqbgW6iAwFJmPdgm6aMeblUvv/DtwGFAKpwC3GmF0VXGulyi908PdP11InNJjnR3SyNh7aBrPHWDelKBJSBwY+7Zkiq1hxD/1YUQ9dA10pb1ZmoIuIDZgCXAqkAPEiMs8Yk1Ci2Wog1hiTLSJ3Aq8Coyuj4Moy+aetbNqXwbQJsdQNC4akxfDpBAgIgsEvgC3Yahh9PoQ18GitVaV4DD3bGkPXHrpS3s2dHnpvINEYkwQgInHACKA40I0xv5Rovwy4viKLrGwf/rGDKb9s57qekdYKiqtmwjf3QYN2MG4O1G3p6RI9oijAjxT10HWWi1JezZ2zXM2B5BLPU5zbTuVWYIGrHSIyUURWiMiK1NRU96usRO/8up1/fpPAkI6NeX5kJ0jdAt/eD60ugFt/qLZhDhRfGVp8UlSvFFXKq7nzE+pqTVjjsqHI9UAs8C9X+40xU40xscaY2IYNPXtHH2MMk3/cxssLNnNl12b8b1wPatgCYMHDEBQGV0+DkNoerdHTQksPuWgPXSmv5s6QSwpQYolBIoG9pRuJyCDgceAiY0xexZRXOYwxvLpwC28v3s61PSN55Zou2AIEEr62xs4ve9UvbiF3tk46KaqBrpRXc6eHHg/EiEgrEQkGxgDzSjYQke7Au8BwY8zBii+z4hhjeOabBN5evJ1xfaJ4tSjM84/B949B404Qe6uny/QKIXqlqFI+pcweujGmUEQmAQuxpi1ON8ZsFJFngRXGmHlYQyy1gM+cy8vuNsYMr8S6y8XuMDz59QY+Wb6bm8+L5qkrOhxfDnfJG5CRAte855drm5eH9tCV8i1uJZcxZj4wv9S2p0o8HlTBdVW43AI798WtZuHGA9w5oA0PD2l/PMwPb4c//wOdR0HL/p4t1IsE2QIIDBCOOZfQraFXiirl1apFV/TIsXxumxHP6uQ0nryiA7ee3+r4TmPg+0eteeaDn/NckV4qNMhGZl4hIUEBJy9SppTyKn4f6DsPHeOWD+NJScvhrXE9uKxz0xMbbP0etv0Ag5+H8Cau36QaCwkuCnQdblHK2/l1oC/dfpg7Z61EgFm39Tn5JhUFubDgEWjQHvrc4ZEavV3ROLqeEFXK+/ltoM+J383jX24gukEY798YS8v6YSc3+vM/kLYLJszz63XNz0ZRkGsPXSnv55eB/r+ft/HaD1u5IKYBU8b3OHlNc4C1c+C3f0GHq6D1RVVfpI8oujpUA10p7+dXgW6M4Y1FW/nvz4lc3b05r17bhUBbqZkZDgf88jwseR2iL4Ar3vRMsT4ipLiHrjNclPJ2fhPoxhhenL+J95bsYEyvFrw4srM1KyMnzRpayU23Gh7aBjt+hR4TYNjrEBjs2cK9XNECXTqGrpT384tAzyu08+jn6/ly9R5u7NeSp6/saIX5kST4ZLS1nnloXatxQBAMeRH63gWi0/DKoidFlfIdPh/oh7LyuH3mSlbuOsrfL23HPZe0tS4Y2vkHzLkeMHDjN9Y65uqM6UlRpXyHTwd6UmoWE6b/xaGsPOIG5dF3+cWwJPN4gwbtYGwc1G/juSJ9XNF6LhroSnk/nw30fek53PD+X+QW2Pnsth50/nqodSehfndbDYJCoedNEBrh0Tp9XaieFFXKZ/hkoB89ls+E9/8iPaeAuIl96bT9PWu8/PovoO1AT5fnV3QMXSnf4XPdruz8Qm6ZEc+uw9m8NyGWTmEZ8NtrcO6VGuaVoHiWi95PVCmv53OBPuWXRNYmp/Gfsd3p16Y+/PCEtWPIi54tzE+F6ElRpXyGzw253F9vOXc1nEzY4kBYbODQVrj4CYiI8nRpfkmvFFXKd/hcoAeFNyQosvPxDTGDof89nivIz+lJUaV8h88FOucMs36pKqEnRZXyHW51u0RkqIhsEZFEEXnUxf4LRWSViBSKyLUVX6byFJ2HrpTvKDPQRcQGTAEuAzoAY0WkQ6lmu4GbgE8qukDlWTrkopTvcGfIpTeQaIxJAhCROGAEkFDUwBiz07nPUQk1Kg/q1iKCiRe2pk+r+p4uRSlVBne6Xc2B5BLPU5zbzpiITBSRFSKyIjU1tTxvoapYSJCNx4adS1gN3zvdolR1406gu1qS0JTnw4wxU40xscaY2IYNG5bnLZRSSp2CO4GeArQo8TwS2Fs55SillCovdwI9HogRkVYiEgyMAeZVbllKKaXOVJmBbowpBCYBC4FNwKfGmI0i8qyIDAcQkV4ikgJcB7wrIhsrs2illFInc+tMlzFmPjC/1LanSjyOxxqKUUop5SE6uVgppfyEBrpSSvkJDXSllPITYky5ppSf/QeLpAK7yvnyBsChCizHV1TH466OxwzV87ir4zHDmR93S2OMywt5PBboZ0NEVhhjYj1dR1WrjsddHY8ZqudxV8djhoo9bh1yUUopP6GBrpRSfsJXA32qpwvwkOp43NXxmKF6Hnd1PGaowOP2yTF0pZRSJ/PVHrpSSqlSNNCVUspP+Fygl3V/U38gIi1E5BcR2SQiG0XkPuf2eiKySES2OX+v6+laK5qI2ERktYh863zeSkSWO495jnPFT78iIhEiMldENju/837V5Lt+wPn3e4OIzBaREH/7vkVkuogcFJENJba5/G7F8h9ntq0TkR5n+nk+Fehu3t/UHxQC/2eMORfoC9ztPM5HgZ+MMTHAT87n/uY+rFU9i7wCvOk85qPArR6pqnJNBr43xpwDdMU6fr/+rkWkOXAvEGuM6QTYsJbm9rfv+0NgaKltp/puLwNinL8mAm+f6Yf5VKBT4v6mxph8oOj+pn7FGLPPGLPK+TgT6we8OdaxznA2mwFc5ZkKK4eIRAKXA9OczwW4BJjrbOKPx1wbuBB4H8AYk2+MScPPv2unQCBURAKBmsA+/Oz7Nsb8BhwptflU3+0I4CNjWQZEiEjTM/k8Xwv0Cru/qa8QkWigO7AcaGyM2QdW6AONPFdZpfg38DBQdLPx+kCac01+8M/vuzWQCnzgHGqaJiJh+Pl3bYzZA7wG7MYK8nRgJf7/fcOpv9uzzjdfC/QKu7+pLxCRWsDnwP3GmAxP11OZROQK4KAxZmXJzS6a+tv3HQj0AN42xnQHjuFnwyuuOMeNRwCtgGZAGNaQQ2n+9n2fzln/ffe1QK829zcVkSCsMJ9ljPnCuflA0X/BnL8f9FR9leA8YLiI7MQaSrsEq8ce4fwvOfjn950CpBhjljufz8UKeH/+rgEGATuMManGmALgC6A//v99w6m/27PON18L9Gpxf1Pn2PH7wCZjzBslds0DbnQ+vhH4uqprqyzGmH8YYyKNMdFY3+vPxpjxwC/Atc5mfnXMAMaY/UCyiLR3bhoIJODH37XTbqCviNR0/n0vOm6//r6dTvXdzgMmOGe79AXSi4Zm3GaM8alfwDBgK7AdeNzT9VTSMZ6P9V+tdcAa569hWGPKPwHbnL/X83StlXT8A4BvnY9bA38BicBnQA1P11cJx9sNWOH8vr8C6laH7xp4BtgMbABmAjX87fsGZmOdIyjA6oHfeqrvFmvIZYoz29ZjzQA6o8/TS/+VUspP+NqQi1JKqVPQQFdKKT+hga6UUn5CA10ppfyEBrpSSvkJDXSllPITGuhKKeUn/h/YxkKMrWj8zgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores3)\n",
    "plt.plot(test_scores3)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "看上去cost和leaky ReLU的cost曲线一模一样，训练出来的效果这么神奇"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# maxout + Sigmoid + Cross Entropy  \n",
    "没有在pytorch中找到maxout的接口"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear + Sigmoid + Cross Entropy  \n",
    "用于对比"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(28*28, 30)\n",
    "        self.fc2 = nn.Linear(30, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = self.fc1(x)\n",
    "        x = F.logsigmoid(self.fc2(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net = Net()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr = 1e-1)\n",
    "loss_scores, train_scores, test_scores = fit(net, criterion, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD6CAYAAACxrrxPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdd3yN1x/A8c+52ZsMKwmx996jRFVLa1bN8uvWgarVQYumtEptapWaLa22KFpK7QoS1CZGkCE7kZ3ce8/vj6sRsi6S3Izzfr3y4j7PeZ7ne0W+98l5zvkeIaVEURRFKf40pg5AURRFyR8qoSuKopQQKqEriqKUECqhK4qilBAqoSuKopQQKqEriqKUEEYldCFENyHEZSHEVSHEx9nsryKE2CuEOCOE2C+E8Mj/UBVFUZTciLzGoQshzIArQFcgCDgBDJZSXsjU5mdgu5RyjRDiaeA1KeWw3M7r6uoqvby8njB8RVGU0sXf3z9SSumW3T5zI45vBVyVUl4HEEJsBHoDFzK1qQeMuff3fcCWvE7q5eWFn5+fEZdXFEVR/iOEuJnTPmO6XNyB25leB93bltm/QL97f+8LOAghXB4lSEVRFOXJGJPQRTbbHu6nGQ90EkKcAjoBwYA2y4mEGC6E8BNC+EVERDxysIqiKErOjEnoQYBnptceQEjmBlLKECnli1LKpsCke9viHj6RlHK5lLKFlLKFm1u2XUCKoijKYzKmD/0EUFMIURXDnfcgYEjmBkIIVyBaSqkHPgFWPU4w6enpBAUFkZKS8jiHKyZibW2Nh4cHFhYWpg5FUUq1PBO6lFIrhBgJ7ALMgFVSyvNCCB/AT0q5DfAGvhJCSOAgMOJxggkKCsLBwQEvLy+EyK6nRylqpJRERUURFBRE1apVTR2OopRqxtyhI6XcCex8aNvkTH/fDGx+0mBSUlJUMi9mhBC4uLignokoiukVuZmiKpkXP+p7pihFQ5FL6IqiKCWV1OsJ+3omqTduFMj5VUJ/iL29PQAhISG89NJLJo5GUZSSJO7XX4n+/nuST54skPMb1YdeGlWqVInNm5/4sUCutFot5ubqW6Aoj0NKWbjdfekp/DcFJ1Wr41FX79RFRxM2cxbWzZpi17NH/seHSug5CgwMpEePHpw7d47Vq1ezbds2kpKSuHbtGn379mXmzJkA7N69mylTppCamkr16tX5/vvvsbe3x8fHh99//53k5GTatWvHsmXLEELg7e1Nu3btOHLkCL169WLcuHEmfqeKUrxIKVnne5PZu6+wYHBTOtUqhDktB2bCvukZL60e4xQhvmXQJ9hQqfwuTm5ZQOsBE/IvvnuKbEL//PfzXAi5m6/nrFfJkSk96z/WsadPn+bUqVNYWVlRu3ZtRo0ahY2NDdOmTWPPnj3Y2dnx9ddfM2fOHCZPnszIkSOZPNkwEGjYsGFs376dnj17AhAbG8uBAwfy7X0pSmmRqtXx2ZZz/OQXhKWZhom/nuWvsR2xtSzAVHbnLOyfQYpXZzaFVyE0LoWmlcvgaGNp9CnMb4RhH3iIlHZ1ONm0P2612xVIqEU2oRc1Xbp0wcnJCYB69epx8+ZNYmNjuXDhAu3btwcgLS2Ntm3bArBv3z5mzpxJUlIS0dHR1K9fPyOhDxw40DRvQlHykVanZ53vTW5GJRXaNf1vxnA2OI5RT9egfQ1XBi33Zf7eAD7pXtfoc0gp2ewfxHkjbhiF1PPG5bcpa+ZAz9v/IyzdljlDmvBc/QpGX0939y6B/QcgK1em8oJVfHN2Aa9V9cz7wMdQZBP6495JFxQrq/u/ZJmZmaHVapFS0rVrV3788ccH2qakpPDee+/h5+eHp6cnU6dOfWD2q52dXaHFrSgFITYpjVE/nuJQQCQO1ubZFnwqCHZW5iwd2oxuDSoCMLCFJysP3aBvU3fqVHDM8/jkNB0f/3qGradDsLcyR5NH4APYjQfn+YSR2Di7sWVgE2qWdzA63tSrVwkaMZK04GBsF3zJq/uGczn6Mo1cG1HZsbLR5zFWkU3oxUGbNm0YMWIEV69epUaNGiQlJREUFES5cuUAcHV1JSEhgc2bN6sRM0qJcenOXYav9edOXApf92vIwJb5n5iM9XH3Ovx1MYyJv55l6dDmhlKC6Ylo0hKztI1MTGLKdn+uRSQwvqMXA1t6InL5KJIpsciVmwl3a82YnsNACEgLIfx2joc8IP3UGeKnfgU2NsR9M5bh4TPRSR2LuyzmKY+nHvMd504l9Cfg5ubG6tWrGTx4MKmpqQBMmzaNWrVq8dZbb9GwYUO8vLxo2bKliSNVlPwREptM/yVHsbUyY+PbbWhWuaxJ4ylrZ8mk5+sy7ud/afXlXnprDjPD4jtsRNoD7RKEYJx7BW6VsYAysCwClu3M4aT3vL1TR5d/bYHbsLDvY8V3tQJ80y+N6LA5VHeqzvyn51PFscpjncsYea5YVFBatGghH17g4uLFi9Sta3xfmFJ0qO9d6fDWWj8OBUSw+4NOVHaxNXU4gKFP/O/zIVT0m0G9wLWElW1OYMXuD7T5Ie04+3SXGF62LeWsbPI8p/3FSKouPUlM+xokNazzWHHprSyI61AfaWmBhcaCZ72exc7iybtbhRD+UsoW2e1Td+iKohhl1/k7/HUhjI+718mSzHXaNDbsGkHd5ERampcxbKzWGRr1f6Dd7sDdHAw6mPXkCeEQcz3rSgsPsRUa3rDypLzm/jMtAXSJuAzBftDqbco/N53yZvcrf56LPMe+HesYVGcwI1tPzPN96lNSuP5NL4SXF22W/ILG0vjRLKamErqiKHlKSNUyddt56lRw4I0OD1bVjIsNZMLWARwlmbJ6ybY4KJOeAqc3wK2j0G0GWo2G2X6zWX9xPc7WzliZZRrJnRoPKbGAxtBPnYsoDfyVEsKcBGiaeQkdMwvovRiaDn2gvVavxeeoD642roxqOsqo9xq5ZCnpt29TefXqYpXMQSV0RVEyCYlNJjldl2X76iOB3LmbwqIhzbAwu18x5MrVPxh98EPCNJLh5dqyMvIEc1v15vM2k2Hv53BkPjHhZ5ng5sKx6PO87PUC4+q+goXGHKSEQ7MhYCPU6QF9l4JV7iNIAmICGL1vNK+bh/JRy49oXbH1gw3iHqyRsjtwNxejL/JNp29wsMz+3PrkZNJD7wCgDQ8jatUqnHr3xq5N62zbF2UqoSuKQrpOz5c7L/L9kcAc2wxpXZnmVe4/BN19aBqfXt2IPbCqxUSaNBhCut8cvj//Pb2q96J5Vx8uOZZj9LlviYwM5YuoaPrcWAL7ljx44s6T4KnxoMm7tFTNsjX58YUf+ejQR0w/Nj3P9gAd3DvwbJVns92XfPYsQSNHoQ0Ly9hm5uREuY8+NOrcRY1K6IpSykUlpDLyh1McvR7FsDZVaOGVdeSKlbkZT9cxDMfVadNYvO1lVsRfohGWzH1hLeXKNwDgncbvsCtwFz5HfXiz4Zv4BKzB0b4cq6v0paHdw2vLA85Vwb35I8XrZOXE4qcXczj4MInpWYcnZqbRaOjo3jHbmi+xv23hzpQpmLu6UvHLLxH3uldsGjfC3Nn5kWIqKoxK6EKIbsB8DCsWfSelnPHQ/srAGqDMvTYf31sUQ1GUIuxccBxvr/MnIiGVOQMa82IzDwDi0+KZ5z+PqJQo0Osg4iJ/nkoG4I42kfMaHf0sKzLxxV+wzNRNYmthy8TWExn590gmHp5Is3LNmO09G1cb13yN20xjRifPTrm20SUkErlwAdEhO4jOsi+BpKO+2LZpg/vcOZiXNe3wy/ySZ0IXQpgBi4GuGBaMPiGE2CalvJCp2afAT1LKJUKIehhWN/IqgHgLnJmZGQ0bNkSr1VK1alXWrVtHmTJlcj3G3t6ehISEB7a9+uqr9OjR44EJRdm1UxRT2Xo6mI9+OUNZW0s2v9OWRh73/5/PPzmfn6/8TI2yNQwjUBIjwNwaMNyxTa74DP2fnZvteTt5duK1Bq+BhFFNR2FhVvhrzaYFBnJ75EjSbgRiVa1atg9bXd56C7fR7yNKUMVTY95JK+CqlPI6gBBiI9AbyJzQJfDfvFsnICQ/gyxMNjY2nD59GoBXXnmFxYsXM2nSJBNHpSj5R6vTM3PXZZYfvE6rqs58+3IzXO3vjzo5G3GWny7/xJC6Q/i46ouwpB3U7wv9Vhh9jbHNxxZE6EZJOHiQ4HHjEWZmVP5uBXb36iuVBsYscOEOZJ7sGnRvW2ZTgaFCiCAMd+fGjQ8q4tq2bUtwcHDG61mzZtGyZUsaNWrElClTTBiZojye2KQ0Xlt9guUHr/NK2ypseLP1A8lcq9fi4+uDm40bIxuPgO1jwNIWnjPuAaQpSSmJXL6C22+/g4W7O16bN5eqZA7G3aFnNzD04eH/g4HVUsrZQoi2wDohRAMppf6BEwkxHBgOULlyHvUf/vjYULYyP1VoCN1n5N0O0Ol07N27lzfeeAMw1D0PCAjg+PHjSCnp1asXBw8epGPHjvkbo6IUkIuhdxm+zo+wuFRm9mvEgJZZK/5tuLiBS9GXmOM9B/sL2+DmYegxD+zLmSBi4+mTkgiZOIn4P//E8fnnqTjtCzS2RWMma2EyJqEHAZm/8x5k7VJ5A+gGIKU8KoSwBlyB8MyNpJTLgeVgmPr/mDEXqOTkZJo0aUJgYCDNmzena9eugCGh7969m6ZNmwKQkJBAQEBAjgk9u6fqajFlpaDtOBPKhmM3s9136lYsjjbm7HwhlRqXRsOF9Af2XySVxdyhEzY8s38+hPwLnq2h2SuFEfoj0cbEEPrpZ+gTDaNc0oOCSA8JodyE8Ti//nqp/VkzJqGfAGoKIaoCwcAgYMhDbW4BXYDVQoi6gDUQ8USRGXknnd/+60OPi4ujR48eLF68mPfffx8pJZ988glvv/22UedxcXEhJiYm43V0dDSurvn7pF9RMtPpJdN3XCBdL/HKps5KlzpuzCi/F/vdX4KTJzjd7zndKZKYImJxQsMkvSMCLXi2gm4zjBofXthiN/1Ewt692DRrBkJg4eFBhalTse/Q3tShmVSeCV1KqRVCjAR2YXjAvUpKeV4I4QP4SSm3AeOAFUKIMRi6Y16Vpqr6lU+cnJxYsGABvXv35t133+W5557js88+4+WXX8be3p7g4GAsLCwySuU+zNvbm3nz5vHKK69gaWnJ6tWr6dy5cyG/C6U0+ftSOPHxt/m6UzrP16ietYH/Mji8Beq/CL0XgaUdWr2W+Sfns/r8apqXb87sTrNxsXEp/OAfgdTpiNm0Cdu2bajy/femDqdIMWq8zr0x5Tsf2jY5098vACXuo7Fp06Y0btyYjRs3MmzYMC5evJixIpG9vT3r16+nXLlyJCUl4eHhkXHc2LFjGTt2LP7+/jRv3hwzMzOqV6/O0qVLTfVWlFJg26E1uFZfw6ehgtSz0fRNeGjSjdBAVx9o9z4IQWxKLBMOTsA31JfBdQYzoeUELDSFP8TwUSUcOIA2NJTyn3xs6lCKHFU+V8kX6ntnWut3jmNu2C6c9eDpXIMTd68x2L0zE2oMMNRNAcODTedqAFyOvszofaMJTwrnszaf0bfm49X7NoVbb75F6pUr1Ph7b4kaQ24sVT5XUUqaAzPh0g7SkXxjlsgPFmnUTjFnRo8f8HKvxTz/eay5sIaDcVdwsnLKcvj12Os4WjqyuttqGrk1MsEbeDxpN2+SePgwrqNGlspknhf1L6IoxU1COByYSbRLdcbZgx9pNIqrQBnnL6jhWQ+A8S3H08C1Aduvb0dmU2S8ZpmajG42Gjdbt8KO/onE/LgRzM0p079/3o1LIZXQFaW4ObmWC+bwgZst0Wnx9K4wjvUX3djYo+YDzbpV7Ua3qt1MFGT+0ycnE/vbbzh0fQaLHAYjlHZFbzySoig502nZfuZ7/udeCakxZ0TtuWw5VJE6FRxoXbV4Vgg0hjY6mtvD30YfF4fzyy+bOpwiS92hK0oxodVrmbtnFGsdNDS3r0Id+8l8/ks4Dd2dWDqseYmdTJN8/jxBo0ahi4qm0syvsW2R7fNABZXQFaVYiEmJYcLBCRy7c4xByTpu8jFLT4TzYlN3vnyxIdYWZqYOsUAk+flx6403MXN2psqGDdg0qG/qkIo01eXyEHt7+0I/NjAwEBsbG5o0aUK9evX43//+R3p6ep7HNGjQIMt2b29vMg8HzamdUnxcjr7M4B2DORnmj09EFO4xndh7OY7PetRj9oDGJTaZ69PSCP30M8zLlaPq5p9VMjeCukMvIqpXr87p06fR6XR07dqVn376iZdVX2HpEbAH4kOzbN4QdIY5MX9hK6z4MrEiXRJu8rzwZt3rrWhXo2SXkohasYK0wEA8V6zA3KVoz14tKtQdei5yKpfbp08fmjdvTv369Vm+fHmW4yIjI2nbti07duxg2LBhbN26NWPfyy+/zLZt23K8ppmZGa1atcoo26vT6ZgwYUJGHMuWLcvHd6gUCZFXYUM/2Dbyga/k30cxP2ontVOS+O3mFbqFHeGQZUdWjepZ4pN56o0bRC1bjuPzz2P/VAdTh1NsFNk79K+Pf82l6Ev5es46znX4qNVHRrXNrVzuqlWrcHZ2Jjk5mZYtW9KvXz9c7t1BhIWF0atXL6ZNm0bXrl2xt7dn7ty59O7dm7i4OP755x/WrFmT43VTUlI4duwY8+fPB2DlypU4OTlx4sQJUlNTad++Pc8++2yJfQBWKl3abvjzrb/B7v5wvBkHtpIcu4xu9aei7d6SO0CnSl6YmRf96flPQkrJnc99EFZWanr/IyqyCd3UciuXu2DBAn777TcAbt++TUBAAC4uLqSnp9OlSxcWL15Mp06G9Q47derEiBEjCA8P59dff6Vfv36YZzPD7dq1azRp0oSAgABeeuklGjVqlBHHmTNn2Lx5MwBxcXEEBARQq1atbONWZXuLocs7oUKjBxZL1usl20NPobG2YXC7/sWixkp+kFIS+e23JPn6UmHKZMzditfEJ1Mrsgnd2DvpgpJTudz9+/ezZ88ejh49iq2tLd7e3qSkpABgbm5O8+bN2bVrV0ZCBxg2bBgbNmxg48aNrFq1Ktvr/deHHhoaire3N9u2baNXr15IKVm4cCHPPffcA+0DAwOzPY8q21vMxIfB7ePg/ckDm/++cocUi3M0dW5bapK5PjGRkEmfGhap6NGDMgMHmjqkYkf1oefgueeeY9WqVRmLOgcHBxMeHk5cXBxly5bF1taWS5cu4evrm3GMEIJVq1Zx6dIlZsy4X8/91VdfZd68eQDUr5/7k/qKFSsyY8YMvvrqq4w4lixZkjHq5cqVKyQmJuZ4vLe3N+vXr+e/omtr1qxRZXuLsit/ABLqvPDA5mW+e9GYJzG4fnfTxFXI0m7fJnDwEOJ376bchAlUmjUTUQTrsBd1RfYO3dSeffbZbMvlduvWjaVLl9KoUSNq165NmzZtHjjOzMyMjRs30rNnTxwdHXnvvfcoX748devWpU+fPkZdu0+fPkydOpVDhw7x5ptvEhgYSLNmzZBS4ubmxpYtWwC4fPnyA2V7586dy/Dhw7l06RKNGzdGCEGLFi0yPhyUIujSTihTGcrf/6C/HZ3EmZgjWDub4135KRMGVzgSjhwheOw4ADyXLy/1i1Q8CVU+txAkJSXRsGFDTp48iZNT1sp3JUFJ/d4VqNQEmFkNWr4B3e5/6H618yLrg9+hjWddVnYruTX0pZREr/qe8NmzsapeHY/Fi7DMa61h5cnL5wohugHzMaxY9J2UcsZD++cC//1ebwuUk1KWefyQS449e/bw+uuvM3bs2BKbzJVH99eFMP7avJyZ+lRePuzKiUN/ZOzTmodgVy2abtW6mDDC/KeLjyd04iQS9u8H7q00n56Ow3PPUenL6Wjs7EwZXomQZ0IXQpgBi4GuGBaMPiGE2HZvlSIApJRjMrUfBTQtgFiLpWeeeYZbt26ZOgyliNDrJQv/vsrcPVdY5ehPst6Rxq2fo5G4/6N4LtGffxMEnT1LzrOP1GvXCBoxkrSgIMoO6I/GzjCr2tLLC6cX+6qRWPnEmDv0VsBVKeV1ACHERqA3cCGH9oOBKTnsU5RSJSlNyw/HbhGfokWjT6f8xdXIyEiWeTjSOd4fUb8HH3a/X5pBL/UM3P4vjdwa4WpTMkYnxf/9NyETPkRYW1Nl9fequFYBMiahuwO3M70OAlpn11AIUQWoCvydw/7hwHCAyqqvTCkFvtx5kfW+ht/QBpjtY7TFCsNPXSSgsYDG94fmxafF88mhT7gUfYnP2nxmmoDzkdTrifx2CZGLFmHdoAEeCxdgUbGiqcMq0YxJ6Nn9LpTTk9RBwGYppS67nVLK5cByMDwUNSpCRSmG0nXpnLwVxYbj1/hfuypMer4ufPcVqbo68PYByNzFoEvl9t3bjNk/hqD4ICa1nkT/WsV7RR5dQgIhH31Mwt69OPXpQ4WpU9BYW5s6rBLPmIQeBHhmeu0BhOTQdhAw4kmDUpTiSqfXseDUAlafX41e6rGvDb/FwG8bAJt7jTa0zPZYZ2tnVjy7ghYVineXROqNGwSNHEVaYCDlJ06k7LChqo+8kBiT0E8ANYUQVYFgDEl7yMONhBC1gbLA0XyNsJSZN28ew4cPx9bW9rHPMXXqVFasWIGbmxtpaWl89tlnDB48OM9j7O3tGT9+fMa2wMBAevTowblz53JtpxjEpcbx0cGPOBJyhNp2nThzw4Y+Td2pVd4ezm+B8Ivw1Fgwt8pyrLkwp1vVblSwq2CCyPNPwoEDBI+fgDA3p/LKldi1ybZ3VikgeSZ0KaVWCDES2IVh2OIqKeV5IYQP4Cel/K904GBgozTVwPYiQKvVZlun5VHMmzePoUOHPlJC1+l0mJk9WBN7zJgxjB8/noCAAJo3b85LL72EhUXpmEJeWM5HnWfn9Z3opZ6EFC17bu4jUR9JM9u3OHGmFh2ru/B1lxaIpGjYMhGaDYOmJfMXWCklUcuWEzF/PlZ16+C5cCEW7u6mDqvUMSr7SCl3Ajsf2jb5oddT8y8s0wgMDKRbt260bt2aU6dOUatWLdauXYutrS0+Pj78/vvvJCcn065dO5YtW4YQAm9vb9q1a8eRI0fo1asXtWrVYtq0aaSlpeHi4sKGDRsoX748U6dO5caNG4SGhnLlyhXmzJmDr68vf/zxB+7u7vz+++8sWbKEkJAQOnfujKurK/v27WP37t1MmTKF1NRUqlevzvfff4+9vT1eXl68/vrr7N69m5EjRzJo0KBs31PNmjWxtbUlJiaGcuXKce3aNUaMGEFERAS2trasWLGCOnXqFPK/dPH3W8BvfOH7BQKBpZklKek60lLt0ES+y8lULyqVsWZqr/qGrobT60GXCi3eMHXYBUKfmEjIJxOJ370bxxdeoOK0L9DY2OR9oJLviuzU/ztffknqxfwtn2tVtw4VJk7Mtc3ly5dZuXIl7du35/XXX+fbb79l/PjxjBw5ksmTDZ9hw4YNY/v27fTs2ROA2NhYDhw4AEBMTAy+vr4IIfjuu++YOXMms2fPBgwVFfft28eFCxdo27Ytv/zyCzNnzqRv377s2LGD999/nzlz5rBv3z5cXV2JjIxk2rRp7NmzBzs7O77++mvmzJmTEYe1tTWHDx/O9f2cPHmSmjVrUu7eKunDhw9n6dKl1KxZk2PHjvHee+/x99/ZDkpSspGuT2fm8ZlsvLyRNhXbMKvjLPQ6W9p8uZdBrTzxGf7Q6lB6PZxYCVXaQ/l6pgk6n+mTk9HfqyekjYoiZPwEUq9do9yHH+L82quqv9yEimxCNxVPT0/atzfUkhg6dCgLFixg/Pjx7Nu3j5kzZ5KUlER0dDT169fPSOgDM1WFCwoKYuDAgYSGhpKWlkbVqlUz9nXv3h0LCwsaNmyITqejW7duADRs2DDb6om+vr5cuHAhI560tLSM2jIPX/dhc+fOZcWKFVy/fp0///wTMJQA/ueff+jf//4IitTU1BzPkdMPZmn+gZ3jN4eNlzfyav1XGd1sNOYac5YeuEaaTs/QNlUebKzXwZ4pEHsTnikZUzPu7tpN6CefoE9Kythm5uSE54rl2LdXNVhMrcgm9LzupAvKw8lKCEFKSgrvvfcefn5+eHp6MnXq1IySuQB2maYsjxo1irFjx9KrVy/279/P1KlTM/ZZWRkehmk0GiwsLDKupdFo0Gq1WWKRUtK1a1d+/PHHbGO1y2Wq9H996L/++iv/+9//uHbtGnq9njJlynD69Om8/yHIWooXDOV4M39IlSYJaQn8GvArPav1ZFwLQzEpnV6y4dhNWld1plZ5h/uNk2Ng8xtwbS+0eB3qGVeYraiSOh0RCxcStXQZNo0b49Snt2GHENh37IhFpUqmDVABVPncLG7dusXRo4aBOj/++CMdOnTISN6urq4kJCRkLDaRnbi4ONzvPQzKbWWinDg4OBAfHw9AmzZtOHLkCFevXgUMRb6uXLnySOd78cUXadGiBWvWrMHR0ZGqVavy888/A4YPjH///TfHY+3t7alYsSJ79+4FDMn8zz//pEOH0rkk2Pbr20nSJjG4zv0RQweuhHM7OpmRte/CtvdhywjD13JvuHEQes6HHnNBU3wXcpbp6QSNGEnU0mWU6f8SldetpezgwYavQYNUMi9CVEJ/SN26dVmzZg2NGjUiOjqad999lzJlyvDWW2/RsGFD+vTpQ8uW2Y8jBsOwvv79+/PUU0891sISw4cPp3v37nTu3Bk3NzdWr17N4MGDadSoEW3atOHSpUd/rjB58mTmzJmDXq9nw4YNrFy5ksaNG1O/fv0H1judNm0aHh4eGV8Aa9euZdq0aTRp0oSnn36aKVOmUL169UeOobiTUrLp8ibqudSjgev9fvJ1R2/yut0ROhwaCud/g+v7DV+WDvDqDmj+qqlCzjdRq1eTsH8/5SdOpIKPDxpLS1OHpORAlc/NJLtx14pxTP29e2xpSRAfmmczv6jzvHZ0Ej6NR9HX01AFMSQmib/W+PCK2W6o2gn6rwZb5wIOuHClBQVxvUdP7J/qgMfChaYORyEfyucqSomUEgdLOkBc3tUwN7q54GhjTbetH8K9m6BKwCtmkND8Heyfnw5mJevHSUrJHR8fhEZD+UmTTB2OYnEIwGQAACAASURBVISS9T/wCXl5eam789Jkrw/cDYLus8Am5/L9Eenx7D23gCFurbBp0RWANJ2eqdvOU7ZidSb0fLWQAi5c8bt2kXjwEOU/+RiLCsV7BmtpUeQSupSyVA+LK46K5eTgIH/D+PDWb0Pr4Tk2k1Ky2u8btOgZ0OFTcDQMTdx+Mogfkh3Y8LRpp7ZLKYnduBFdfAIub7yOyDRjOH7PHuK2/Z7xG0VmZmXL4jb6fcxdXDK2pV6/QdSypeiTkgFI8vPDql5dyr78csG/ESVfFKmEbm1tTVRUFC4uLiqpFxNSSqKiorAuTpX0dFr4fTQ4VIDOOXclpOpSme47nd+u/kav6r2o4nh/nPnaozep5mZHu+ouOR5f0PTJyYR+Npm727cDkHT8OO6zv0Fjb0/E/AVELV+OeYUKmDk4ZDk27cABEg4dwmPhQmwa1Cd+/35Cxk8AyChxa1m5MhU+n4p4wnIWSuEpUt8pDw8PgoKCiIiIMHUoyiOwtrZ+YLFqk4oJhKTo3Ntc3glhZ2HAWrB2zLZJWGIYY/aP4WzkWd5p/A7vNn43Y9/ZoDhO345lSs96JrvxSA8O5vaoUaRevITbB6MxK+vMnWnTuNF/AJaeniQeOUKZAQMo/+mkbEelJJ8/T9DIUdx8+WUcu3cnbutWVYOlBChSCd3CwqLUTlpR8sGV3fDDAHIu159JrW5Qt1e2u9J16byz5x1CEkKY5z2PLlUeXNtzve9NbCzMeLGZaT7E9Kmp3HrjTbSRkXgs+RYHb28ArGrWJGj0+yQeP06FqVMpOyjnmcQ29etTdfPPBH8whrgtW3Ds2ZOKPp+rGizFXJFK6Iry2NISYcc4cK0FXX1yb6sxA6+nHlxkIpM1F9ZwNfYqCzovoHPlB9f1jEtKZ+u/wfRt6o6TjWmqV0YtX0FaYCCeK797YLq9bbOmVNu2DV1sLFZG3BiZu7hQedVKUi5dxrpBfdXNWQKohK6UDAe+Ngw/fO0PqNLusU9zO/42y/5dRpfKXTKS+ZZTwfzsb1iFMToxnZT0bOq2FJLU6zeIWr4cxx49sq2dYl62LOZlyxp9PmFhgU3DBnk3VIoFldCV4u/OOfhnETQd+kTJXErJ9GPT0QgNH7f6GIA0rZ5pOy5ipgHPsrbYWZrxajsv6ldyyq/oHym+O1OnImxsKP/xR4V+faXoMyqhCyG6AfMxLHDxnZRyRjZtBgBTMXRg/iulzLKqkaLkO70etn9gGEfe9YtHPjwoPoighCAALkdf5kjwET5q+VHGykG7zt8hMiGV1a+1xLt2uXwN3Rjpd+6QduMGAMlnzpJ0r3/c/DHKSiglX54JXQhhBiwGumJYX/SEEGKblPJCpjY1gU+A9lLKGCFE4f/PV0qnk6sh6AT0WfrI0+5TtCkM2TGEmNT7FSXru9RnUJ37i4WsO3qTKi62dKzpll8RGy09NJTrPXpm1B4HsGnWjDIDivcC0krBMeYOvRVwVUp5HUAIsRHoDVzI1OYtYLGUMgZAShme34EqShbxYbBnquEBZ+PsV2zKza7AXcSkxjCl7RSqOhkeItZzqYe5xvBjcenOXY4HRjPp+bpoNIX/wPDO9OlInQ7PZUvR3CuVbN2wIUKjauop2TMmobsDtzO9DgIenh5XC0AIcQRDt8xUKeWf+RKhouRk10RITzaUp32MERqbLm+imlM1+tXsl+0Ij/W+N7Ey1/BS88Ifnhi/dy8Je/ZSbvw47Dt1KvTrK8WTMR/12f2kPDzQ1xyoCXhjWCz6OyFEluIYQojhQgg/IYSfmjykPJGre+HcZugwFlxrPvLh5yPPczbyLANrD8w2mcenpPPbyWB6Nq5EWbvCLRerT0zkzrTpWNWqhfMrrxTqtZXizZg79CDAM9NrDyAkmza+Usp04IYQ4jKGBH8icyMp5XJgORjK5z5u0EoplJYENw6ALt3w+q/J4FwdOozJaCKl5FT4KaJTDDNFzTXmtKnYBmvzrGUJNl3ehI25DT2r98z2cr+dCiYxTcewAhqeKKUk6dhxdHfjsuxL2Ps32tBQ3GfPRliYZqy7UjwZk9BPADWFEFWBYGAQ8PAIli0Y7sxXCyFcMXTBXM/PQJVSLPoGbHwZws/f3ybMYNhvYHE/WW+/vp2Jhx9curCOcx3md55PJfv7q+rEpcax88ZOelXvhYNl1jonO8+GMuOPSzStXIbGnjlXYXxc+uRkQj/9jLs7duTYpuyQIdg2a5rv11ZKtjwTupRSK4QYCezC0D++Skp5XgjhA/hJKbfd2/esEOICoAMmSCmjCjJwpZS49jf8/Bogof8acKlh2G7rDI73k3RsSiyzTsyikVsjJreZDMCNuzfw+ceHQdsH8U2nb2hVsRUAW65uIVWXysDaD06N1+klc/66zOJ912hauQxLhzbP97eTHhzM7ZGjSL10CbfR72P/9NNZ2ggzMyxL4apQypMrUisWKUoGKeHoIkPXilsdGLQBnKvl2HzKP1PYenUrm3psorZz7YztgXGBjN43mpt3b+LhYHi4GZ4UTh3nOsx5agWf/HqWq+EJAKSk6wiNS2FQS08+710fK/P8XQc00fcYwWPGILVa3L+ZpR52Ko9FrVikFC9pSfD7+3D2Z0MBrT5LwMo+x+b+Yf78GvArr9V/7YFkDuDl5MWG5zew9N+lhCcbRtPWd6lPa5de9Fx4mOjENLrWK4/m3oPRjrXc6NfMPV/rmkgpiVm3jrCvZ2Lp5YXHooVG1VpRlEelErpiGpEBcPtY1u1SwvHlcOcsPP0ZPDUu1yGJ6bp0vjj6BZXsKvFO43eybWNnYUdLp1cIEykARMSn8smPAbjaW/HLu+1o4F5w0/hlWhqhn00mbutW7J/pQqUZMzCzz/nDSVGehEroSuE78xNsGwXalOz3WznBkE1Q67k8T7XmwhquxV1j0dOLsLWwzbI/JV3HZ1vO8bN/0APb21ZzYdGQprjYWz3WWzBW5NKlxG3diuvIkbi+966aFKQUKJXQlcKj08KeKYa+8SrtDROCLLKpv23jnGsXy39u373N0n+X0rVKVzp5Zu2PvhOXwtvr/fn3dizvP12DAS0No281QlDRybrAy8WmXr9O5IrvcOzVE7eRIwr0WooCKqErjyvI33CXnfgIE8R0aZASC62Gw3Nfgtnjj7H+rzKiucacj1pmrTwYfjeFXosOk5iqZenQ5nRrULiLHEspuTNlKhpbW8p/pCojKoVDJXTl0Z1aD9vHgH0FqPPCox1btSM0ePGJQ9gVuIsjIUf4uNXHlLcrn2W/z/YLxCals2VEe+pVyn6ZuYIU99sWkk6coILP5w8sxKwoBUkldMUgKRrO/XJ/JmZOws7D6fVQtRP0X/3IFQ7zQ2xKLDOOzzBURqydtSjXgSsRbD8TyphnahVaMk8PDyd+91+g0xoWzl66zFAZ8aWXCuX6igIqoSsAoWcMMzHjbhnRWEDbkfDM52BW+P99rsZcZfS+0cSlxfHtM99ipnlwrHhymo5Pt5ylmpsd73jnPG49PyWdPEXQ6PfRRURmbNM4OVFh6hT1EFQpVCqhl3ZnN8PWkWBTFl7fDW61c29vZgGWdoUT20P23tzLxMMTsTG3YeWzK6nnUi9Lm4V/B3A7Opkf32qT7xODshPz00/c+WIaFhUr4vnLZiw9DQ9ehbU1GsvCLeqlKCqhlybaVEPJ2dM/gNTf25YCldvCgLVgX3TWJUnX6ZnxxyX2XQrnmwGNCdP58uHBD2ng0oBWdmN4ZUkEado/shyXqtXTr5kHbasXbL+1TEvjzvQvid20CbsOHXCf/Q1mToW/LJ2iZKYSemkRfwc2DYOg49BoEDjce5BoXwFavgnmReduMiohlRE/nMT3ejRlbS0YtGIvZWvPpYFLQ1zjRzP/cAQda7lRt2LWwlr2lua80t6rQOPTRkQQNPoDkk+exOXNN3AbMwZhVvC/DShKXlRCLy4ubIPwC9nvq9YZKj+05sitY3B9n+HvUg/+ayD1ruFBZv2+TxxOuk7PxhO3iUpIfeJzZSYlbPYPIiIhlTkDGvN0nXL03jSaKG0CVy90wzcmkvHP1mJE5xoFOo480deXJD//rDv0emJ/+QXd3bu4z5mN4/PPF1gMivKoVEIvDq7vh5+G5bz/wNeGBZLb3pu84vst7P70frcKGKoUDv0FKjR44nCiElJ5b8NJjt2IfuJzZcfT2YbN77SlkUcZ/MP8idYcpp5NL26kV+K7/zWkS92swxTzi9Trifx2CZGLFuXYxrJKFTyXLcW6Tp0Ci0NRHodK6EVdegpsHwtlq8K7/2SdWZmWAFvehd2TIPRfQ92TM5ugbk/o/S1YZeqWeMw7Wq1Oj/5eUc5Ld+7y7vqTRCakMm9gE3o3qZT7wY9JCEG6Lh2foz5UsqvE6t6TsDG3KdC7cl1CAiEffUzC3r049eljGKVilX1pgIKeZaooj0Ml9KLu8FyIvmZYzMEya60SrBxgwDo49A38Pd2w7elPocM4eMIhc1JK1vneZPqOi6Rq79/tV3KyLvCiVqm6VKb+M5XrcddZ3GVxtnVa8vV6N24QNHIUaYGBlJ84kbLDhqqkrRQ7KqEXZZEBcHgONOwP1bMuhJBBCOg4ASq3AyR4dXjiS6ek65i89Rw/+QXxVE1X2lQzjBqxNNPwYjP3Ai1qdSfxDmP2jeFc1DlGNBlBR4+OBXYtgIQDBwgePwFhbk7llSuxa/PwGuiKUjwYldCFEN2A+RhWLPpOSjnjof2vArMwLFEHsEhK+V0+xln6SGmYXm9hY6h7Ygyv9o99uVtRSfxw/BbpOsOd+PEb0ZwNjmPU0zUY80wtNJrCuVs9GXaSMfvHkKJNYX7n+TxdOZcPslzok5OJXrMWXUyMYYOZGU59emNdq1ZGGyklUcuWEzF/PlZ16+C5cCEW7u758TYUxSTyTOhCCDNgMdAVw2LQJ4QQ26SUDw+52CSlHFkAMZZO/26EwEOGioQFPD784JUIRv14isRULdYWhuF3dlZmLB3ajG4NKhbotf8jpeSnyz8x4/gM3B3cWfXcKqqXebxl2NKCggi6t8ybxs4wCUqflkbMjz9S6cvpOHbvjj4xkZBPJhK/ezeOPXpQ8QsfNDbZVH5UlGLEmDv0VsBVKeV1ACHERqA3kMMYOuWJJUUbHnJ6tIJmr+b76WOT0tDde8q52T+Ir/+8RK3yDiwf1oLKLgXbV52dNF0aXx77kl8CfuEp96eY0XEGjpaPV4Ml8ehRgj8Yg5QSzxXLsX/qKeDe2PH3RxM8ZixJJ0+R5OtL6rVrlPvwQ5xfe1X1lyslgjEJ3R24nel1EJBdJ2M/IURH4AowRkp5O5s2ijH++gxS4qDnvCd+sPmwb3ZdZtG+qw9se6FRRWa91Ahby8J/pBKeFM6Y/WM4E3GGtxq+xYgmI7LUZzFW0qlT3HrzLayqVcVj0SIsq1TJ2Gfu5kaVNau58+WXxKxbh5mTE5W/W4Fdu3b59VYUxeSM+QnO7tbl4ZWlfwd+lFKmCiHeAdYAWTo/hRDDgeEAlStXfsRQS4nAI4bytO1HQ/n6+Xrqc8FxfLv/Kl3rleepmq4AlHOw4rn6FUxyh3o6/DRj9o8hMT2R2Z1m86zXs499Lpmezp3JUwyJ+4cfMHPIOotUWFpScepUHLo8g1X1alhUKpghl4piKsYk9CDAM9NrDyAkcwMpZVSmlyuAr7M7kZRyObAcoEWLFg9/KJQesbfgn4WQnpR1342D4FQZOuW9KIKUkvW+NzkbHJexraWXMy8198iSoHV6ycTfzuJsZ8U3LzXGyfbxF5fID38G/snEQxMpb1ue5V2XU7NszSc6X/SaNaQGBOCxeFG2yTwz+6eefBSQohRFxiT0E0BNIURVDKNYBgFDMjcQQlSUUobee9kLuJivUZYkNw7Cz69CWiLYZlNAytza0NWSR0XDhFQt43/6lz/P38HNwQpzjSBdp+cnvyCO3YhmWp8GGQ84Adb73uRMUBzzBzUxeTJP16fz9fGvqV22Nku7LsXJ6snGs6cFBROxaDH2Xbrg0KVLPkWpKMVPngldSqkVQowEdmEYtrhKSnleCOED+EkptwHvCyF6AVogGni1AGMuPqQ03I3rtYbXV3YZpuS71DCUqnWtAUBiqhY7q6zfirjkdGIS07Jsj01O58PN/3I1PIFPX6jLGx2qIoRAr5fM3xvA/L0BBITFM71vQ+ytzIlP0TJr12WequlKr8am72bYe2svkcmRfN7u81yTuZQSmZKS6+gTKSV3vvABjYYKn04qiHAVpdgQUpqm56NFixbSz8/PJNcuNMdXwM7xD26r/Tz0XQbWjuj1krl7rrBo31XmDGhM36YeGc1uRCbSa+Fh4lO12Z66jK0FiwY3o8O9vvDMdp2/w9hNp0lM02VsszLXsHtMR6q4mKaWeWav/fkaoYmh7Oi7I9cHoJErVhC5cBEVfD6nTJ8+WfbL9HTCZs0iZu06yn38ES6vvlqAUStK0SCE8JdStshun5opWlD0evBdAhUaQttRhm3WTlDzWdBouJuSzpiNp9l7KRwnGwt8fr9Ap1rlcLazRErJp1vOAjDrpUaYm2V9YNm2misVnKyzvfRz9Svw5wcd8bt5v3hWvYpORSKZX425il+YH2Oaj8k1mcv0dKLXrkVKSejHn5By4QLlJ0xAWBi6i7TR0YYhiMeOUXbYMJyH5VK8TFFKCZXQC8qNA4YaLH2XQ+OBBMcmM/PPS6SdOAXA+ZC7hMQm49O7Pq2ruvDCgkN8tfMis/o3ZuvpEI5cjeKL3vXp38Izjwtlz9PZFk/nwh9TnpdNlzdhqbGkb43cS/jG792LLiISj0ULSTx+nJi160jy88PSw/DvkXzmDLroaCrO+Crbu3dFKY1UQi8oJ74zPPSs1xuAjcdvse3fEGqWswfA2c6SmS81yqiR8uZT1Vh64Bpd65Vn2o4LNPYsw5DWVXI8fXGUmJ7I79d/5zmv5yhrXTbXtjE//IiFuzv2nTvj8MwzWNerR/SataTduA6ARcWKeCxciE3DJy8HrCglhUroBSEuGC7vhHbvg4WhW2T3+TBaeTmz6e222R4yuktNtp8J4e31/miEYM3rDTArpPophWXH9R0kpicysM7AXNulXr1K0vHjuI0bm7ESUJk+fdSduKLkQS1JXhD8VxtGuLR4DYCbUYlcDouna72cF2awsTTjiz4NkBJeb+9F/Uola33KE3dOsPDUQuo616WRa6Nc28b8uBFhYUGZfv0KKTpFKRnUHXp+06YZEnrNZ6GsFwB/XQgD4Nl6FXI9tHPtcuwZ24mqrqZ/eJlfpJT8cOkHZp2YRWXHyszsODPXWan6xETitmzBoXs3zJ2dCzFSRSn+VELPb5d+h8RwaPVWxqbdF8KoU8HBqMJXNe71sZcEqbpUfI76sO3aNrw9vfmqw1fYW+b+/qLXrkWfmIjzkCG5tlMUJSuV0PPbiZVQpgpUN8xYjEpIxS8wmpGda5g4sMJ1J/EOH+z7gPNR53m38bu80/gdNCLnHj6ZlkbYjBnE/PAj9k8/jXXjxoUYraKUDCqh56ewC3DzCHT1yaiSuPdSOHoJz9bPvbulOLoSc4WYlJgs22NSYvjq+FdZFqlIDw8n7fr1LO2lTkfkkiUk+/nj/MbrlBszRpWzVZTHoBJ6fvJbCWZW0GRoxqa/LoRRycma+pUer753UaTT61hwagGrzq3KsU0VxyoPLFIR//c+QiZMQJ+YmG17YW1NpW++wanHCwUSs6KUBiqh55fUeMMqQw1eBDvD2PLkNB2HAiIY2MKzxNxxxqXG8dHBjzgScoT+tfrzfNXns7QRQlDXuS62FrZIvZ7IpUuJXLAQ63r1KDd+XMZsz8wsPD2xqFDyfotRlMKkEnp+ObMJ0hI4XeElPv/2CHoJyWlaUtL1Jaa7JSr0Bn+/9xIvxCXzln0lnK3PA+ezbRt27099YiJp167h2KsnFX180FhnX65AUZQnpxJ6fpASjn8HFRvz+UlrbkYn0dDdiTI2FjT2KEOrqiVj+N3RSe9Q53ISokVDHK3KGHWMmZMTzkNfpsygQSXmtxRFKapUQs8PN/+BiIsEdZzFqd1xTO5Rj9c7VDV1VPnK/891VP/nFtd7NeWFmT+YOhxFUbKhZormhxPfgbUTy6KaYG2hoV9zj7yPKUZSkxOInz6LqLLmeH+2yNThKIqSA5XQn1R8GFzcRmqDwfx8Joo+TdxxsjHtikD5be9XoygfkY7FhyOwcygZ3UeKUhIZldCFEN2EEJeFEFeFEB/n0u4lIYQUQmRbfL1EOrkW9Fq2WXQnJV3P0DYlq0Lirf07cf/Fl+stKtG67zumDkdRlFzkmdCFEGbAYqA7UA8YLISol007B+B94Fh+B1lk6bTg/z2yWme+PQvNKpehgXvJKKolpSR67VoS3htPeBlo8MUcU4ekKEoejLlDbwVclVJel1KmARuB3tm0+wKYCaTkY3xF25U/4G4wFz0GciMykWFtS8bduUxLI/TjTwj78isC6pdh9eg6VKmqpuIrSlFnTEJ3B25neh10b1sGIURTwFNKuT23Ewkhhgsh/IQQfhEREY8cbJFz4jukoweTL7rjam9J9wYVTR1Rvoj49lvitm7F7p3XmdIjgfa1upo6JEVRjGBMQs9u8HDGytJCCA0wFxiX14mklMullC2klC3c3NyMj7IoigyA6/v5t3wf/G7H83H3ulhb5LxGZnGReu0aUStX4dS7F/4v1ECHpLNnZ1OHpSiKEYxJ6EFA5oUtPYCQTK8dgAbAfiFEINAG2FbiH4z6LkFqLBgT0Ig21Zzp18w972OKOKnXEzplChpbW8p9+CF/3/6binYVqeNcx9ShKYpiBGMS+gmgphCiqhDCEhgEbPtvp5QyTkrpKqX0klJ6Ab5ALymlX4FEXBSEnAb/7zni+DxB6Q5M69OwRMyCjPvtN5L9/Ck3fhxpjjYcDTlKZ8/OJeK9KUppkGdCl1JqgZHALuAi8JOU8rwQwkcI0augAyxy9DrY/gFpVs68d6cH73rXKBGLUmgjIwmfOQub5s0p068fR0OOkqpLzSh9qyhK0WfU1H8p5U5g50PbJufQ1vvJwyrCTnwHIadY6PgRTs6uvOdd3dQRPbGUS5cIGjESfXIyFadOQWg0/H37bxwsHWhWvpmpw1MUxUhqpuijuBsCe78g3qMTC8Mb8Vq7qsX+QWjcjh0EDhqM1Gqpsn4dVjVrotVrORB0gE4enbDQlKxZr4pSkqniXNn5YRBc3591u14LGjO+tX0PGwuKdc0WqdMRMW8eUSu+w6ZZMzzmz8PczQ0pJRsubiAuNU51tyhKMaMS+sOSouHKn1CtE1RolGV3gqc33/+QRt+mxbdmiy4ujuBx40k8fJgygwZSYeJEhKUlabo0ph+bzq8Bv9LRoyPeHt6mDlVRlEegEvrDbv4DSOj0MVRpm2X3psM3SEm/UORrtpwOP41vqC8AdoERuPgb1vIUEsofvIhVZDwBw7twp0sFuGhYSu5Q0CHORJ5heKPhjGgyItdFnRVFKXpUQn9Y4GEwtwH3rA8D9XrJet+bNKtchvqVimbNFikl6y+uZ7bfbHRSh/cZPW/9qcdCd79NpCNMG2JGgMsBOH0gY7uDpQNzvefyTJVnTBC5oihPSiX0hwUeBs9WYG4FGBKkVm+YGPvPtShuRCby/kDT1jWRUqKV2izbU7WpfHVkGjtubOfpSt58cMyZxB0/Y9umDRW+mYmZk+FDqKZGQ7tsxpYLIdRduaIUYyqhZ5YUDWHnoPNEAG5FJfHuBn/Oh9zNaOJsZ9qaLZeiLzFu/zhuxd96YLtLnGTMFh1DQ2AoAHtJBJxfe41y48YizNW3WlFKOvVTntmto4AErw4cCohg5A+nAHi/S00szQx3tC28nE02VPGPG38w+chkHK0cH+jjdrwQRN3129Gk60gd1g0PF8Pyd9Z162LfqZNJYlUUpfCphJ5Z4GEwt2bdbVem7DhOzXIOLP9fc6q42OV6WEBMAGciztC3Zt8C67JYcHIBK86uoFm5Zsz2no2rjSsAsZs3EzptIZYeHnh8uxiratUK5PqKohR9KqFnFniIBLdmTNkRwNN1yjF/UFPsrHL/J0rVpfLBvg+4FX+Lfbf38dVTX+Fg6ZCvYR0MOsiKsyvoU6MPk9tMxsLMMFwyNSCA0KmfY9eqFe7z52HmkL/XVRSleFFPwP6THIO8c46tsVVxsbdi9oAmeSZzgBVnVnAr/hYDag3gSPARhuwYwsWoi8SkxGT5SkpPeuSwktKTmO47nWpO1R5I5lKvJ3Tq55jZ2VHpm1kqmSuKou7QM9w8ikCyNbYakwfVM2rS0PXY66w8t5IXqr3AZ20/o1vVbow/MJ4B2wdk295CY8GHLT9kYO2BRlcwXHpmKSGJIazutjojmQPE/foryf7+VJw+DXNntXCzoigqoWdIvLIfc2mBQ4029GiU9ygWKSU+vj7YmtsyocUEAFpWaMlPPX5i/+396KQuyzGHgg8x/dh0LkZfZFLrSViaWeZ6jSsxV1h3fh19a/SlefnmGdu1UVGEzfoGmxbNcXrxxUd8p4qilFSlOqF/u/8qt6IM3SBvXdhDpKzJlD7NjLp73nZtG/5h/kxpOwUXG5eM7eXtyjOwzsBsjxlUZxCLTy9m+ZnlXIy6SD2XLGttP+Bk+EkcLB0Y02gUkUuXkh4cDEDKlSvok5KoOHWqqlWuKEqGUpvQb0cnMfPPyzjZWFDbPITqumsk1B1NZRdbo47/7epv1ChTgxdrGn+HrBEaRjUdRV3nusz1n8uhoEO5trcws8Cn9hjiho8m+dQpzN3cQAgQgvIffohVjRpGX1tRlJLPqIQuhOgGzAfMgO+klDMe2v8OMALQAQnAcCnlhXyONV/5Xo8C4Ke321L71DQ4YUHjniONOjZFm8KZiDMMqTPksYYpPlPlmRyn12tjYtDHxwOQHhREyOhPZGzXpgAAEuBJREFU/t/evUdHVd0LHP/+8uQREiA8Ql4EFEEQFQwI7YXLswVE6FpXvFqtuoCrS8XSam2p3IsK2pZiq1ZBofaBIiKitRRRSwleQC8xIDQICRIBk0ACgUASEvKc3/1jBhqSCUzChGFmfp+1sphzzj7n/Pba8efOnn32obKsjIQXnid6woRm38sYEzwumtBFJBRYDIzH+X7RDBFZ2yBhr1TVV13lpwC/Ba7o7JN+sJjO7SPo01Fg10roPxWiunl07u7ju6lx1DAkbojX4lFVTr7+Okd/vQjq/jX+Hp6YSMqqt2jTt6/X7mWMCUye9NCHAjmqegBARFYBU4FzCV1VS+uVbw+oN4NsDdsOnGBoSmdC9qyBqlIYMtPjczMKMxCEQd0HeSUWR2UlBfPmUbr2b0SNG0v0+PHOAyGhRI0ccW4NFmOMuRBPEnoCkFdvOx+4uWEhEXkYeBSIAK7oNyPkFVeQf/IMM7+d4nylXLcBkDzM4/MzCjPo17kf0RHRlxxLzZEj5M96hMqsLLrO/iGxDzyAhNjjAcaY5vMkc7ibRtGoB66qi1X1KuBnwH+7vZDI/SKyXUS2FxUVNS9SL0o/WAzAqKhvoHA3DJ3p/LLRA1V1VWQWZXpluKX88885eNs0qnNzSVyymC4PPmjJ3BjTYp5kj3wgqd52InDkAuVXAd9zd0BVl6lqqqqmdu3a1fMovSz9wAk6tgun59crIaIDDHT/IJA7mUWZVDuqLymhqyrFK94kd/oMQmNiSFm9mg6jR7f4esYYA54l9Aygj4j0EpEI4A5gbf0CItKn3uYtwH7vheh92w6e4NaECmTv+3DjnRAZ5fG52wu3IwiDuzd+AYanSteu5egzzxA1YgQpq98msnevFl/LGGPOuugYuqrWisgs4GOc0xb/qKp7RGQ+sF1V1wKzRGQcUAOcBO5tzaAvxeFTZ8grruDBNksgrA2MeKxZ52ccvbTx89qTJzn6q4W0vfFGEhe/bEMsxhiv8WgeuqquB9Y32Dev3ufZXo6r1aQfOMHUkE+JL06HSc9BhziPzz07fn57X8+HaBo69txz1JWVEff005bMjTFeFXQZ5Z9fHeTJiBVo/E2QOr1Z5+4u2k1VXRWp3VNbdO+K7dspefc9Yu+7lzZ9r2nRNYwxpilB9+j/TftfJIbTyK0vQMjF3zx0/MxxDpYcBGD9wfUIct5CWQ05qqqo3L0brXOcf0CVwgULCI+Pp8tDD11SHYwxxp2gSugFX2cypW4DXybfzXU9rr9o+U/yPmHOljmU15Sf23dd7HXERLp/0Kc6L4/8WY9QtW9fk9dMfPUVQtp5tl6MMcY0R1Al9OJPXiFWQ4ke9/gFyznUwbLMZSzetZhrO1/Ljwb/6Nxa5L1i3M9IKf/sMw7/+FFUlfiFvyIsrvESvGFdYom86qpLr4gxxrgRPAm9upyU/L/yacS3GZ2c0mSx8ppy5m6dy8bcjUzuPZknhz9Jm7A2F7x06YcfcvixnxB5VW8SX36ZiJ49vRy8McZcXNAk9PLtq2iv5RRec3eTZb4p/YbZabM5VHqIx1Mf5wf9f3DR9cZVlaIXf0dk3770fOMNQqMu/EJpY4xpLcGR0FWp2baMLEcyA4Z9x22RrYe38tP//SmhIaG8Ov5VhvXwbG2Xim3bqD50iPhfL7RkbozxqeCYtpifQcfSbP4aPpGBiR0bHT7bM4+PimfV5FUeJ3OAkytXEtqpEx2++11vRmyMMc0WFAm9Lv33nNa21PS/rdEQiqqyYNsCIkIjeGXcKyREJXh83ZrCQso2ptHxtv8gJDLS22EbY0yzBH5Cz9+B7P0La+pG8O8DG89Q+eDgB6QXpDN78Gy6tmvegmGnVq8GVTr+p/t3iBpjzOUU2Al95wr400ROhcayMnQqw3rHnne4pKqERRmLGNhlINOumdasS2t1NSffeYeokSOJSEz0ZtTGGNMiAfmlqKOujry3fkjPnBUc6zKMO4ofYEC/XkSEnf//r+d3PE9JVQlLxy8l1IOnRusr27iRuqLjdPr+nd4M3RhjWiwge+j7tqyhZ84K/lT7XYbnP8yBikgmX3/+gz47j+3k3f3vcve1d9Ovc79mXV+rqzm+ZAnhycm0HzHCm6EbY0yLBWQPPWTfB5RqO4Y8sIRN7doRERZCXMy/Hg6qcdQw///mE9c+jodubP66Kif+vJyq/TkkLlliKyYaY64YgZfQ62pJPPYJm0NSmZjUxW2R5XuWk3Mqh5fGvES78Oatq1Kdl8fxxYvpMH48HcbYW4aMMVeOwOte5qXTvq6EnM4j3R7OL8tn6T+XMjZ5LKOSRjXr0qpK4fwFSGgo3ec+4YVgjTHGezzqoYvIBOBFnG8sek1Vf9Xg+KPATKAWKAKmq+o3Xo7VI47sD6jVMKpSGveeVZVn058lREKYM3SOR9er2LmTyi/3AFBTUED5li10f+IJwuM8fzGGMcZcDhdN6CISCiwGxuN8YXSGiKxV1b31iu0EUlW1QkQeBH4NXP7J2arU7V3HVsd19IpvnHD3n9rP1sNbefSmR4lrf+GErKqc+P1rFD3/PKie299uyBA63fV9r4dujDGXypMe+lAgR1UPAIjIKmAqcC6hq+qmeuW3AU2vgNWaju0lvPQbNjjGcXePDo0Op+WmIQi3XnXrBS/jqKjgyNy5lH34EdGTJtJtzhwk3Ll8bmhMjH0Raoy5InmS0BOAvHrb+cDNFyg/A/jwUoJqsez1KEKapvJUt6hGh9Ny07i+6/V0aev+y1IAra0ld/oMzmRm0u0nj9F5xoyLrrhojDFXAk+6mu6ymbrZh4jcDaQCi5o4fr+IbBeR7UVFRZ5H6ansdRyI7Ed0lwQiw85/UKiwvJCs4izGJI+54CVOvvkmZ3btIv6XvyB25kxL5sYYv+FJQs8HkuptJwJHGhYSkXHAXGCKqla5u5CqLlPVVFVN7dq1eeumXFRJPhTs4u91qfSNcz/cAjAmqemEXlNQwLEXf0f7kSOInjLFu/EZY0wr8yShZwB9RKSXiEQAdwBr6xcQkUHAUpzJ/Jj3w/TAPucozzvlN3Btj+hGh9Py0ugV04uUmJQmL1H47LPgcBA3b571zI0xfueiCV1Va4FZwMdAFrBaVfeIyHwROduNXQREAe+IyC4RWdvE5VpP9joqY3pzQOPp2/38HnpJVQk7CndcsHdelpbG6X9spMvDD9liW8YYv+TRPHRVXQ+sb7BvXr3P47wcV/OcOQWHtvJ1yj1wFPo1mOGy5fAWarWW0cnun+x0lJdTuOAZIvv0Ifa++y5DwMYY432B8ej//g3gqGVr6M10iAwjoWPb8w6n5abRpW0XBnYZ6Pb0opdepraggISVK89NTzTGGH8TGBOqs9dBVHfSSpPoG9fhvPHvqroqPj38KaOSRhEijatbmZVF8Rtv0PH222k3eNDljNoYY7zK/xN6bRXk/AO9ZiJ7j55uNMPl9T2vU1FbwaRekxqdqnV1FDz5FKEdO9LtsUcvV8TGGNMq/D+hH9wM1acpThpHWWUt/erNcMkrzWNp5lLG9xzPkLghjU49+fbbVGZm0n3OzwiNibmcURtjjNf5f0LPXgcRUewMvR6Afq4euqryTPozhIWEuV2IqyxtE8ee+w3tvzWc6MmTL2vIxhjTGvw7oTsczvnnV49jY04JUZFhXJ/o7Gl/dOgjPjvyGY8MeoRu7bqdO0UdDoqWLCH/oYeITEmhxy9/aXPOjTEBwb9nuRzeDqeP4ug7iQ1/O8qovl2JDAulrLqMhZ8vZEDsAKZ1n8DRRYtwlJYBUJ2bS0V6OtFTbqXH/PmEtGlzkZsYY4x/8N+ErgppCyAyhsy2N3P89B7G9+8OwF/2/4UTlSd4Kfkn5N5+BzWFhYR16uQ8LzyM7j+fQ6d77rGeuTEmoPhvQs9c7fxC9Jbf8OHXZwgPFUZ2DaUyL5cN297kriNJhL8wD+3QgZQ3V9D2hht8HbExxrQq/0zoFcXw8ROQkAo3TWfDbzczvfIrCsc/Dg4HZ18O12bwYBJffIEwby8EZowxVyD/TOj/eArOnIR73ifneAXHDx/jli1v0WbAANYPcpBblsvPRsyj07jvIBERvo7WGGMuC/+b5ZK7Db5YDsMehLiB/H1vITP2rCP8TAXhc2ezLHE/8dPuovOkyZbMjTFBxf8S+vH9ENsHRv0cgK/+voXv5GYQO/0+3nPsAGDaNdN8GaExxviE3w251PacRG3EjXAgj+LTVUzc8GfOxHYj5oH/4t0PbmVk4kjio+J9HaYxxlx2fpfQS95/n2OLnju3nQzU/WIh//PFMxRXFnNn3zt9F5wxxviQ3yX0DmPHEtGzJ8dKq3j6b3tIvLYzOeEryD6YzezBsxkeP9zXIRpjjE94NIYuIhNEZJ+I5IhIo4VRRGSkiHwhIrUicpv3w/yXiJQUosaO5amTXdh5VRybk5aTX5bP4rGLmTnQXupsjAleF03oIhIKLAYmAv2BO0Wkf4NiucB9wEpvB+jO2n8eYcv+4wwemE1l3RlW3rKSEYkjLsetjTHmiuXJkMtQIEdVDwCIyCpgKrD3bAFVPeQ65miFGM9TcqaGBeuyGJgYTUHNDob3GH7BFz8bY0yw8GTIJQHIq7ed79rnE7/ffIDi8ioeGNeWgvIjjElu+sXPxhgTTDzpobsblNaW3ExE7gfuB0hOTm7JJZg15mpu6tmJrMo1CMLIxJEtuo4xxgQaT3ro+UBSve1E4EhLbqaqy1Q1VVVTu7ZwfZU24aGM7teNTXmbGNRtELFtY1t0HWOMCTSeJPQMoI+I9BKRCOAOYG3rhnVhh08fJrs424ZbjDGmnosmdFWtBWYBHwNZwGpV3SMi80VkCoCIDBGRfGAasFRE9rRm0J/kfQLA6KTRrXkbY4zxKx49WKSq64H1DfbNq/c5A+dQzGWRlpvG1R2vJjm6ZePwxhgTiPxuca6SqhJ2HN1hvXNjjGnA7xL65vzN1GmdjZ8bY0wDfpfQo8KjGJ00mv6xDR9WNcaY4OZ3i3ONTh7N6GQbbjHGmIb8rodujDHGPUvoxhgTICyhG2NMgLCEbowxAcISujHGBAhL6MYYEyAsoRtjTICwhG6MMQFCVFv0ropLv7FIEfBNC0/vAhz3Yjj+IhjrHYx1huCsdzDWGZpf756q6vaFEj5L6JdCRLaraqqv47jcgrHewVhnCM56B2Odwbv1tiEXY4wJEJbQjTEmQPhrQl/m6wB8JBjrHYx1huCsdzDWGbxYb78cQzfGGNOYv/bQjTHGNOB3CV1EJojIPhHJEZE5vo6nNYhIkohsEpEsEdkjIrNd+zuLyAYR2e/6t5OvY/U2EQkVkZ0iss613UtE0l11fltEInwdo7eJSEcRWSMi2a42Hx4kbf1j1+/3lyLyloi0CbT2FpE/isgxEfmy3j63bStOv3PltkwRGdzc+/lVQheRUGAxMBHoD9wpIoH46qJa4DFVvRYYBjzsquccYKOq9gE2urYDzWwgq972QuB5V51PAjN8ElXrehH4SFX7ATfgrH9At7WIJAA/BFJV9TogFLiDwGvvPwMTGuxrqm0nAn1cP/cDrzT3Zn6V0IGhQI6qHlDVamAVMNXHMXmdqhao6heuz2U4/wNPwFnX5a5iy4Hv+SbC1iEiicAtwGuubQHGAGtcRQKxztHASOAPAKparaqnCPC2dgkD2opIGNAOKCDA2ltVNwPFDXY31bZTgdfVaRvQUUR6NOd+/pbQE4C8etv5rn0BS0RSgEFAOtBdVQvAmfSBbr6LrFW8APwUcLi2Y4FTqlrr2g7E9u4NFAF/cg01vSYi7QnwtlbVw8BzQC7ORF4C7CDw2xuabttLzm/+ltDFzb6AnaYjIlHAu8CPVLXU1/G0JhGZDBxT1R31d7spGmjtHQYMBl5R1UFAOQE2vOKOa9x4KtALiAfa4xxyaCjQ2vtCLvn33d8Sej6QVG87ETjio1halYiE40zmb6rqe67dR8/+Ceb695iv4msF3wamiMghnENpY3D22Du6/iSHwGzvfCBfVdNd22twJvhAbmuAccBBVS1S1RrgPeBbBH57Q9Nte8n5zd8SegbQx/VNeATOL1HW+jgmr3ONHf8ByFLV39Y7tBa41/X5XuCvlzu21qKqP1fVRFVNwdmuaap6F7AJuM1VLKDqDKCqhUCeiPR17RoL7CWA29olFxgmIu1cv+9n6x3Q7e3SVNuuBe5xzXYZBpScHZrxmKr61Q8wCfgK+BqY6+t4WqmO/4bzT61MYJfrZxLOMeWNwH7Xv519HWsr1X8UsM71uTfwOZADvANE+jq+VqjvjcB2V3u/D3QKhrYGngaygS+BN4DIQGtv4C2c3xHU4OyBz2iqbXEOuSx25bbdOGcANet+9qSoMcYECH8bcjHGGNMES+jGGBMgLKEbY0yAsIRujDEBwhK6McYECEvoxhgTICyhG2NMgLCEbowxAeL/AVKV+T7aBdEWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(test_scores, label='linear')\n",
    "plt.plot(test_scores1, label='ReLU')\n",
    "plt.plot(test_scores2, label='leaky ReLU')\n",
    "plt.plot(test_scores3, label='paramter ReLU')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
